From e85d910a0f9934db1685101f8fdb746b6bb471e7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 9 Nov 2022 16:40:46 +0000 Subject: [PATCH 001/116] Initial experimental implementation of PEP 669. --- Include/cpython/code.h | 1 + Include/internal/pycore_code.h | 2 ++ Include/internal/pycore_interp.h | 7 +++++++ Objects/frameobject.c | 27 ++++++++++++++++++++++++++- Python/bytecodes.c | 6 ++++++ Python/pystate.c | 6 +++++- 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ebac0b12a461bc..da05f316226e69 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -98,6 +98,7 @@ typedef struct { PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ + uint64_t _co_instrument_version; /* current instrumentation version */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0af240ca362103..81f0dd3dd4811d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -466,6 +466,8 @@ typedef struct _PyShimCodeDef { extern PyCodeObject * _Py_MakeShimCode(const _PyShimCodeDef *code); +extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ae2a3d3b13cfa9..1b06933263eef7 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -19,6 +19,7 @@ extern "C" { #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_instruments.h" // PY_INSTRUMENT_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct type_cache @@ -85,6 +86,8 @@ struct _is { PyInterpreterState *next; + uint64_t instrument_version; + struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ @@ -188,6 +191,10 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; + uint8_t instrumented[PY_INSTRUMENT_EVENTS]; + PyObject *instrument_callables[PY_INSTRUMENT_EVENTS]; + bool f_opcode_trace_set; + /* The following fields are here to avoid allocation during init. The data is exposed through PyInterpreterState pointer fields. These fields should not be accessed directly outside of init. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 287d5bd9ad1afd..4861f80a0ca88e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ static PyMemberDef frame_memberlist[] = { {"f_trace_lines", T_BOOL, OFF(f_trace_lines), 0}, - {"f_trace_opcodes", T_BOOL, OFF(f_trace_opcodes), 0}, {NULL} /* Sentinel */ }; @@ -104,6 +103,31 @@ frame_getback(PyFrameObject *f, void *closure) return res; } +static PyObject * +frame_gettrace_opcodes(PyFrameObject *f, void *closure) +{ + PyObject *result = f->f_trace_opcodes ? Py_True : Py_False; + return Py_NewRef(result); +} + +static int +frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignored)) +{ + if (!PyBool_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "attribute value type must be bool"); + return -1; + } + if (value == Py_True) { + f->f_trace_opcodes = 1; + _PyInterpreterState_GET()->f_opcode_trace_set = true; + } + else { + f->f_trace_opcodes = 0; + } + return 0; +} + // Given the index of the effective opcode, scan back to construct the oparg // with EXTENDED_ARG. This only works correctly with *unquickened* code, // obtained via a call to _PyCode_GetCode! @@ -846,6 +870,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_globals", (getter)frame_getglobals, NULL, NULL}, {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, + {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, {0} }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index da3ab9126eeeea..a30bfacb6e5e44 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -110,6 +110,12 @@ dummy_func( inst(RESUME, (--)) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->instrument_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } diff --git a/Python/pystate.c b/Python/pystate.c index 04db1fb419af62..1c92ce5d954458 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -309,7 +309,11 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - + for(int i = 0; i < PY_INSTRUMENT_EVENTS; i++) { + interp->instrumented[i] = 0; + interp->instrument_callables[i] = Py_NewRef(Py_None); + } + interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 6283ee898bf205d8a8103977bdf6d67a77d8f4c1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 18 Nov 2022 12:32:59 +0000 Subject: [PATCH 002/116] First draft implementing small subset of PEP 669. --- Include/cpython/pystate.h | 1 - Include/internal/pycore_instruments.h | 80 ++++ Include/internal/pycore_interp.h | 21 +- Include/internal/pycore_opcode.h | 20 +- Include/internal/pycore_pystate.h | 4 - Include/opcode.h | 5 + Lib/opcode.py | 8 + Lib/profile.py | 4 +- Lib/test/test_monitoring.py | 271 ++++++++++++ Makefile.pre.in | 2 + Objects/codeobject.c | 1 + Python/bytecodes.c | 185 +++++---- Python/ceval.c | 570 ++----------------------- Python/clinic/instrumentation.c.h | 196 +++++++++ Python/generated_cases.c.h | 182 ++++---- Python/instrumentation.c | 573 ++++++++++++++++++++++++++ Python/legacy_tracing.c | 302 ++++++++++++++ Python/opcode_targets.h | 10 +- Python/pystate.c | 12 +- Python/specialize.c | 1 + Python/sysmodule.c | 9 + 21 files changed, 1748 insertions(+), 709 deletions(-) create mode 100644 Include/internal/pycore_instruments.h create mode 100644 Lib/test/test_monitoring.py create mode 100644 Python/clinic/instrumentation.c.h create mode 100644 Python/instrumentation.c create mode 100644 Python/legacy_tracing.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 70c23427807e52..99bc0a89dc0e5f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -74,7 +74,6 @@ typedef struct _PyCFrame { * discipline and make sure that instances of this struct cannot * accessed outside of their lifetime. */ - uint8_t use_tracing; // 0 or 255 (or'ed into opcode, hence 8-bit type) /* Pointer to the currently executing frame (it can be NULL) */ struct _PyInterpreterFrame *current_frame; struct _PyCFrame *previous; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h new file mode 100644 index 00000000000000..4c3a1dbcb3a4cb --- /dev/null +++ b/Include/internal/pycore_instruments.h @@ -0,0 +1,80 @@ + +#ifndef Py_INTERNAL_INSTRUMENT_H +#define Py_INTERNAL_INSTRUMENT_H + + +#include "pycore_bitutils.h" // _Py_popcount32 +#include "pycore_frame.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define PY_MONITORING_TOOL_IDS 8 + +/* Require dynamic instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 + + +#define PY_MONITORING_EVENT_C_RETURN 9 +#define PY_MONITORING_EVENT_PY_THROW 10 +#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 +#define PY_MONITORING_EVENT_C_RAISE 13 +#define PY_MONITORING_EVENT_PY_UNWIND 14 + +/* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ +#define PY_MONITORING_EVENTS 15 + +/* Temporary and internal events */ + +// #define PY_INSTRUMENT_PEP_523 50 +/* #define PY_INSTRUMENT_JIT_API 17 -- Reserved */ + +typedef uint32_t _PyMonitoringEventSet; + +typedef uint16_t _PyMonitoringSet; + +typedef struct _pytoolset { + uint8_t tools; +} _PyMonitoringToolSet; + +/* Reserved IDs */ + +#define PY_INSTRUMENT_PEP_523 5 +#define PY_INSTRUMENT_SYS_PROFILE 6 +#define PY_INSTRUMENT_SYS_TRACE 7 + + +/* API functions */ + +PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); + +void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); + +extern int +_Py_call_instrumentation(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_arg(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + +extern void +_Py_call_instrumentation_exc(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUMENT_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1b06933263eef7..1a606a72ae2fd1 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -19,7 +19,7 @@ extern "C" { #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_instruments.h" // PY_INSTRUMENT_EVENTS +#include "pycore_instruments.h" // PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct type_cache @@ -74,6 +74,9 @@ struct _Py_long_state { int max_str_digits; }; +struct _instrumentation_tool { + PyObject *instrument_callables[PY_MONITORING_EVENTS]; +}; /* interpreter state */ @@ -86,7 +89,7 @@ struct _is { PyInterpreterState *next; - uint64_t instrument_version; + uint64_t monitoring_version; struct pythreads { uint64_t next_unique_id; @@ -191,9 +194,19 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - uint8_t instrumented[PY_INSTRUMENT_EVENTS]; - PyObject *instrument_callables[PY_INSTRUMENT_EVENTS]; + _PyMonitoringToolSet monitoring_tools_per_event[PY_MONITORING_EVENTS]; + _PyMonitoringEventSet events_per_monitoring_tool[PY_MONITORING_TOOL_IDS]; + _PyMonitoringEventSet monitored_events; + /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ + int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; + /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ + struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; bool f_opcode_trace_set; + bool sys_profile_initialized; + bool sys_trace_initialized; + Py_ssize_t sys_profiling_threads; + Py_ssize_t sys_tracing_threads; /* The following fields are here to avoid allocation during init. The data is exposed through PyInterpreterState pointer fields. diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 3f44511240aa14..86e7d7bb9511b4 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -136,6 +136,11 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [IMPORT_STAR] = IMPORT_STAR, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, @@ -473,11 +478,11 @@ static const char *const _PyOpcode_OpName[263] = { [237] = "<237>", [238] = "<238>", [239] = "<239>", - [240] = "<240>", - [241] = "<241>", - [242] = "<242>", - [243] = "<243>", - [244] = "<244>", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", [245] = "<245>", [246] = "<246>", [247] = "<247>", @@ -566,11 +571,6 @@ static const char *const _PyOpcode_OpName[263] = { case 237: \ case 238: \ case 239: \ - case 240: \ - case 241: \ - case 242: \ - case 243: \ - case 244: \ case 245: \ case 246: \ case 247: \ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3d6d400f74dd1d..66cbc96016cb13 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -132,10 +132,6 @@ PyAPI_FUNC(void) _PyThreadState_DeleteExcept( static inline void _PyThreadState_UpdateTracingState(PyThreadState *tstate) { - bool use_tracing = - (tstate->tracing == 0) && - (tstate->c_tracefunc != NULL || tstate->c_profilefunc != NULL); - tstate->cframe->use_tracing = (use_tracing ? 255 : 0); } diff --git a/Include/opcode.h b/Include/opcode.h index c18d9c057b849b..70f8b1ea9a3939 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -120,6 +120,11 @@ extern "C" { #define DICT_UPDATE 165 #define CALL 171 #define KW_NAMES 172 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index 0ee75958508ac7..fb240a7472ab86 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,14 @@ def pseudo_op(name, op, real_ops): hasconst.append(172) +# Instrumented instructions +def_op('INSTRUMENTED_RESUME', 240) +def_op('INSTRUMENTED_CALL', 241) +def_op('INSTRUMENTED_RETURN_VALUE', 242) +def_op('INSTRUMENTED_YIELD_VALUE', 243) +def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) + + hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 diff --git a/Lib/profile.py b/Lib/profile.py index 453e56285c510c..747ae117dae437 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,7 +187,7 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ - + print(event, file = sys.stderr) if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] @@ -289,7 +289,7 @@ def trace_dispatch_c_call (self, frame, t): def trace_dispatch_return(self, frame, t): if frame is not self.cur[-2]: - assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3]) + assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3], t) self.trace_dispatch_return(self.cur[-2], 0) # Prefix "r" means part of the Returning or exiting frame. diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py new file mode 100644 index 00000000000000..4e3428effafc7f --- /dev/null +++ b/Lib/test/test_monitoring.py @@ -0,0 +1,271 @@ +"""Test suite for the sys.monitoring.""" + +import sys +import unittest +import enum +import operator +import functools +import types +import collections + +def f1(): + pass + +def f2(): + len([]) + sys.getsizeof(0) + +def gen(): + yield + yield + +def g1(): + for _ in gen(): + pass + +TEST_TOOL = 4 + +class MonitoringBaseTest(unittest.TestCase): + + def test_has_objects(self): + m = sys.monitoring + m.events + m.use_tool + m.free_tool + m.get_tool + m.get_events + m.set_events + # m.get_local_events + # m.set_local_events + m.register_callback + # m.insert_marker + # m.remove_marker + # m.restart_events + m.DISABLE + + def test_tool(self): + sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") + sys.monitoring.free_tool(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + sys.monitoring.set_events(TEST_TOOL, 19) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 19) + sys.monitoring.set_events(TEST_TOOL, 0) + + +class MonitoringCountTest(unittest.TestCase): + + def check_event_count(self, func, event, expected): + + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + self.assertEqual(counter.count, 0) + counter.count = 0 + func() + self.assertEqual(counter.count, expected) + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + counter.count = 0 + func() + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_start_count(self): + self.check_event_count(f1, E.PY_START, 1) + + def test_resume_count(self): + self.check_event_count(g1, E.PY_RESUME, 2) + + def test_return_count(self): + self.check_event_count(f1, E.PY_RETURN, 1) + + def test_call_count(self): + self.check_event_count(f2, E.CALL, 3) + + def test_c_return_count(self): + self.check_event_count(f2, E.C_RETURN, 2) + + +E = sys.monitoring.events + +SIMPLE_EVENTS = [ + (E.PY_START, "start"), + (E.PY_RESUME, "resume"), + (E.PY_RETURN, "return"), + (E.PY_YIELD, "yield"), + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), + (E.JUMP, "jump"), + (E.BRANCH, "branch"), + (E.RAISE, "raise"), + (E.PY_UNWIND, "unwind"), + (E.EXCEPTION_HANDLED, "exception_handled"), +] + +SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + +def just_pass(): + pass + +just_pass.events = [ + "py_call", + "start", + "return", +] + +def just_raise(): + raise Exception + +just_raise.events = [ + 'py_call', + "start", + "raise", + "unwind", +] + +def just_call(): + len([]) + +just_call.events = [ + 'py_call', + "start", + "c_call", + "c_return", + "return", +] + +def caught(): + try: + 1/0 + except Exception: + pass + +caught.events = [ + 'py_call', + "start", + "raise", + "exception_handled", + "return", +] + +def nested_call(): + just_pass() + +nested_call.events = [ + "py_call", + "start", + "py_call", + "start", + "return", + "return", +] + +PY_CALLABLES = (types.FunctionType, types.MethodType) + +class MonitoringEventsBase: + + def gather_events(self, func): + events = [] + for event, event_name in SIMPLE_EVENTS: + def record(*args, event_name=event_name): + events.append(event_name) + sys.monitoring.register_callback(TEST_TOOL, event, record) + def record_call(code, offset, obj): + if isinstance(obj, PY_CALLABLES): + events.append("py_call") + else: + events.append("c_call") + sys.monitoring.register_callback(TEST_TOOL, E.CALL, record_call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + events = [] + try: + func() + except: + pass + sys.monitoring.set_events(TEST_TOOL, 0) + #Remove the final event, the call to `sys.monitoring.set_events` + events = events[:-1] + return events + + def check_events(self, func, expected=None): + events = self.gather_events(func) + if expected is None: + expected = func.events + self.assertEqual(events, expected) + + +class MonitoringEventsTest(unittest.TestCase, MonitoringEventsBase): + + def test_just_pass(self): + self.check_events(just_pass) + + def test_just_raise(self): + try: + self.check_events(just_raise) + except Exception: + pass + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + + def test_just_call(self): + self.check_events(just_call) + + def test_caught(self): + self.check_events(caught) + + def test_nested_call(self): + self.check_events(nested_call) + +UP_EVENTS = (E.C_RETURN, E.C_RAISE, E.PY_RETURN, E.PY_UNWIND, E.PY_YIELD) +DOWN_EVENTS = (E.PY_START, E.PY_RESUME) + +from test.profilee import testfunc + +class SimulateProfileTest(unittest.TestCase, MonitoringEventsBase): + + def test_balanced(self): + events = self.gather_events(testfunc) + c = collections.Counter(events) + self.assertEqual(c["c_call"], c["c_return"]) + self.assertEqual(c["start"], c["return"] + c["unwind"]) + self.assertEqual(c["raise"], c["exception_handled"] + c["unwind"]) + + def test_frame_stack(self): + self.maxDiff = None + stack = [] + errors = [] + seen = set() + def up(*args): + frame = sys._getframe(1) + if not stack: + errors.append("empty") + else: + expected = stack.pop() + if frame != expected: + errors.append(f" Popping {frame} expected {expected}") + def down(*args): + frame = sys._getframe(1) + stack.append(frame) + seen.add(frame.f_code) + def call(code, offset, callable): + if not isinstance(callable, PY_CALLABLES): + stack.append(sys._getframe(1)) + for event in UP_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, up) + for event in DOWN_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, down) + sys.monitoring.register_callback(TEST_TOOL, E.CALL, call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + testfunc() + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(errors, []) + self.assertEqual(len(seen), 9) + self.assertEqual(stack, [sys._getframe()]) + + diff --git a/Makefile.pre.in b/Makefile.pre.in index 2c0ff3d1c7b9a6..a03a506330675b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,6 +395,8 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/instrumentation.o \ + Python/legacy_tracing.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc1db72977aa01..7657b1322df401 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -338,6 +338,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; + co->_co_instrument_version = 0; /* not set */ co->co_weakreflist = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a30bfacb6e5e44..d0ccb70b40f0da 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -13,6 +13,7 @@ #include "pycore_code.h" #include "pycore_function.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -107,15 +108,34 @@ dummy_func( inst(NOP, (--)) { } - inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + inst(INSTRUMENTED_RESUME) { + /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->instrument_version) { + assert(oparg < 2); + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } } + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } + } + + inst(RESUME, (--)) { + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + ERROR_IF(err, error); + err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -200,7 +220,6 @@ dummy_func( } inst(BINARY_OP_MULTIPLY_INT, (left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -212,7 +231,6 @@ dummy_func( } inst(BINARY_OP_MULTIPLY_FLOAT, (left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -226,7 +244,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_INT, (left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -238,7 +255,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_FLOAT, (left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -251,7 +267,6 @@ dummy_func( } inst(BINARY_OP_ADD_UNICODE, (left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -269,7 +284,6 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -299,7 +313,6 @@ dummy_func( } inst(BINARY_OP_ADD_FLOAT, (left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -313,7 +326,6 @@ dummy_func( } inst(BINARY_OP_ADD_INT, (left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -327,7 +339,6 @@ dummy_func( inst(BINARY_SUBSCR, (container, sub -- res)) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -373,7 +384,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -398,7 +408,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_TUPLE_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *tuple = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -423,7 +432,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *dict = SECOND(); DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -501,7 +509,6 @@ dummy_func( inst(STORE_SUBSCR, (v, container, sub -- )) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -519,7 +526,6 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); PyObject *value = THIRD(); @@ -545,7 +551,6 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *dict = SECOND(); PyObject *value = THIRD(); @@ -623,20 +628,27 @@ dummy_func( assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; } + // stack effect: (__0 -- _0) + inst(INSTRUMENTED_RETURN_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, val); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(RETURN_VALUE); + } + // stack effect: (__0 -- ) inst(RETURN_VALUE) { PyObject *retval = POP(); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); frame = cframe.current_frame = pop_frame(tstate, frame); @@ -793,7 +805,7 @@ dummy_func( if (retval == NULL) { if (tstate->c_tracefunc != NULL && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -835,6 +847,16 @@ dummy_func( Py_DECREF(v); } + // stack effect: ( -- ) + inst(INSTRUMENTED_YIELD_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, val); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(YIELD_VALUE); + } + // stack effect: ( -- ) inst(YIELD_VALUE) { // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -846,8 +868,6 @@ dummy_func( PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1062,7 +1082,6 @@ dummy_func( inst(UNPACK_SEQUENCE) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1140,7 +1159,6 @@ dummy_func( inst(STORE_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1271,7 +1289,6 @@ dummy_func( inst(LOAD_GLOBAL) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1331,7 +1348,6 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL_MODULE) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; @@ -1351,7 +1367,6 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL_BUILTIN) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1716,7 +1731,6 @@ dummy_func( inst(LOAD_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1777,7 +1791,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1802,7 +1815,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_MODULE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1827,7 +1839,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1866,7 +1877,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1888,7 +1898,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_CLASS) { - assert(cframe.use_tracing == 0); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *cls = TOP(); @@ -1911,7 +1920,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_PROPERTY) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -1950,7 +1958,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *owner = TOP(); @@ -1991,7 +1998,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2020,7 +2026,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2072,7 +2077,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2109,7 +2113,6 @@ dummy_func( inst(COMPARE_OP) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2123,7 +2126,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2154,7 +2156,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2186,7 +2187,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int invert = cache->mask; @@ -2599,7 +2599,6 @@ dummy_func( inst(FOR_ITER) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2619,7 +2618,7 @@ dummy_func( goto error; } else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); } _PyErr_Clear(tstate); } @@ -2634,7 +2633,6 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER_LIST) { - assert(cframe.use_tracing == 0); _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2657,7 +2655,6 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER_RANGE) { - assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2680,7 +2677,6 @@ dummy_func( } inst(FOR_ITER_GEN) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)TOP(); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2822,7 +2818,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_VALUES) { /* Cached method object */ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2847,7 +2842,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2873,7 +2867,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_NO_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2891,7 +2884,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2932,11 +2924,24 @@ dummy_func( kwnames = GETITEM(consts, oparg); } + // stack effect: (__0, __array[oparg] -- ) + inst(INSTRUMENTED_CALL) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function); + ERROR_IF(err, error); + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + } + // stack effect: (__0, __array[oparg] -- ) inst(CALL) { _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); @@ -2989,16 +2994,26 @@ dummy_func( } /* Callable is not a normal Python function */ PyObject *res; - if (cframe.use_tracing) { - res = trace_call_function( - tstate, function, stack_pointer-total_args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - function, stack_pointer-total_args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + if (!PyFunction_Check(function) && !PyMethod_Check(function)) { + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); + } + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3093,7 +3108,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TYPE_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *obj = TOP(); @@ -3111,7 +3125,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_STR_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3181,7 +3194,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_O) { - assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3215,7 +3227,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_FAST) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3255,7 +3266,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3294,7 +3304,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LEN) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); @@ -3324,7 +3333,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_ISINSTANCE) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); @@ -3357,7 +3365,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LIST_APPEND) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); @@ -3520,6 +3527,10 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(INSTRUMENTED_CALL_FUNCTION_EX) { + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + // error: CALL_FUNCTION_EX has irregular stack effect inst(CALL_FUNCTION_EX) { PyObject *func, *callargs, *kwargs = NULL, *result; @@ -3542,8 +3553,30 @@ dummy_func( } } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func); + ERROR_IF(err, error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); @@ -3718,7 +3751,6 @@ dummy_func( inst(BINARY_OP) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3741,7 +3773,6 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); - assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); diff --git a/Python/ceval.c b/Python/ceval.c index bff8b5c4d488cb..b3695ac7af1d98 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -14,6 +14,7 @@ #include "pycore_code.h" #include "pycore_function.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -101,13 +102,6 @@ #define HEAD_UNLOCK(runtime) \ PyThread_release_lock((runtime)->interpreters.mutex) -/* Forward declarations */ -static PyObject *trace_call_function( - PyThreadState *tstate, PyObject *callable, PyObject **stack, - Py_ssize_t oparg, PyObject *kwnames); -static PyObject * do_call_core( - PyThreadState *tstate, PyObject *func, - PyObject *callargs, PyObject *kwdict, int use_tracing); #ifdef LLTRACE static void @@ -187,19 +181,16 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) PyErr_Restore(type, value, traceback); } #endif -static int call_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static int call_trace_protected(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static void call_exc_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *); -static int maybe_call_line_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, int); -static void maybe_dtrace_line(_PyInterpreterFrame *, PyTraceInfo *, int); -static void dtrace_function_entry(_PyInterpreterFrame *); -static void dtrace_function_return(_PyInterpreterFrame *); + +static void monitor_raise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); @@ -228,20 +219,6 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" -#ifndef NDEBUG -/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and - PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen - when a thread continues to run after Python finalization, especially - daemon threads. */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif - #ifdef HAVE_ERRNO_H #include @@ -699,8 +676,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ - assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -708,7 +683,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ opcode = _Py_OPCODE(*next_instr); \ PRE_DISPATCH_GOTO(); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -779,7 +753,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define PREDICT(op) \ do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \ + opcode = _Py_OPCODE(word) \ if (opcode == op) { \ oparg = _Py_OPARG(word); \ INSTRUCTION_START(op); \ @@ -881,49 +855,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Shared opcode macros */ -#define TRACE_FUNCTION_EXIT() \ - if (cframe.use_tracing) { \ - if (trace_function_exit(tstate, frame, retval)) { \ - Py_DECREF(retval); \ - goto exit_unwind; \ - } \ - } - -#define DTRACE_FUNCTION_EXIT() \ - if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ - dtrace_function_return(frame); \ - } - -#define TRACE_FUNCTION_UNWIND() \ - if (cframe.use_tracing) { \ - /* Since we are already unwinding, \ - * we don't care if this raises */ \ - trace_function_exit(tstate, frame, NULL); \ - } - -#define TRACE_FUNCTION_ENTRY() \ - if (cframe.use_tracing) { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = trace_function_entry(tstate, frame); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ - goto error; \ - } \ - } - -#define TRACE_FUNCTION_THROW_ENTRY() \ - if (cframe.use_tracing) { \ - assert(frame->stacktop >= 0); \ - if (trace_function_entry(tstate, frame)) { \ - goto exit_unwind; \ - } \ - } - -#define DTRACE_FUNCTION_ENTRY() \ - if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ - dtrace_function_entry(frame); \ - } - #define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) @@ -942,63 +873,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); -static int -trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - if (tstate->c_tracefunc != NULL) { - /* tstate->c_tracefunc, if defined, is a - function that will be called on *every* entry - to a code block. Its return value, if not - None, is a function that will be called at - the start of each executed line of code. - (Actually, the function must return itself - in order to continue tracing.) The trace - functions are called with three arguments: - a pointer to the current frame, a string - indicating why the function is called, and - an argument which depends on the situation. - The global trace function is also called - whenever an exception is detected. */ - if (call_trace_protected(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Trace function raised an error */ - return -1; - } - } - if (tstate->c_profilefunc != NULL) { - /* Similar for c_profilefunc, except it needn't - return itself and isn't called for "line" events */ - if (call_trace_protected(tstate->c_profilefunc, - tstate->c_profileobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Profile function raised an error */ - return -1; - } - } - return 0; -} - -static int -trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *retval) -{ - if (tstate->c_tracefunc) { - if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - if (tstate->c_profilefunc) { - if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - return 0; -} - static _PyInterpreterFrame * pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) { @@ -1078,7 +952,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * strict stack discipline must be maintained. */ _PyCFrame *prev_cframe = tstate->cframe; - cframe.use_tracing = prev_cframe->use_tracing; cframe.previous = prev_cframe; tstate->cframe = &cframe; @@ -1113,8 +986,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } - TRACE_FUNCTION_THROW_ENTRY(); - DTRACE_FUNCTION_ENTRY(); + /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -1206,59 +1078,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int case DO_TRACING: #endif { - assert(cframe.use_tracing); - assert(tstate->tracing == 0); - if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { - int instr_prev = _PyInterpreterFrame_LASTI(frame); - frame->prev_instr = next_instr; - NEXTOPARG(); - // No _PyOpcode_Deopt here, since RESUME has no optimized forms: - if (opcode == RESUME) { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - /* Call tracing */ - TRACE_FUNCTION_ENTRY(); - DTRACE_FUNCTION_ENTRY(); - } - else { - /* line-by-line tracing support */ - if (PyDTrace_LINE_ENABLED()) { - maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); - } - - if (cframe.use_tracing && - tstate->c_tracefunc != NULL && !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - _PyFrame_SetStackPointer(frame, stack_pointer); - - err = maybe_call_line_trace(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, instr_prev); - // Reload possibly changed frame fields: - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; - // next_instr is only reloaded if tracing *does not* raise. - // This is consistent with the behavior of older Python - // versions. If a trace function sets a new f_lineno and - // *then* raises, we use the *old* location when searching - // for an exception handler, displaying the traceback, and - // so on: - if (err) { - // next_instr wasn't incremented at the start of this - // instruction. Increment it before handling the error, - // so that it looks the same as a "normal" instruction: - next_instr++; - goto error; - } - // Reload next_instr. Don't increment it, though, since - // we're going to re-dispatch to the "true" instruction now: - next_instr = frame->prev_instr; - } - } - } + assert(0); NEXTOPARG(); PRE_DISPATCH_GOTO(); // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: @@ -1343,12 +1163,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - - if (tstate->c_tracefunc != NULL) { - /* Make sure state is set to FRAME_UNWINDING for tracing */ - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame); - } + monitor_raise(tstate, frame, next_instr-1); exception_unwind: { @@ -1367,8 +1182,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_UNWIND(); - DTRACE_FUNCTION_EXIT(); + monitor_unwind(tstate, frame, next_instr-1); goto exit_unwind; } @@ -1399,6 +1213,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyException_SetTraceback(val, Py_None); Py_XDECREF(tb); Py_XDECREF(exc); + monitor_handled(tstate, frame, next_instr-1, val); PUSH(val); JUMPTO(handler); /* Resume normal execution */ @@ -1414,7 +1229,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); _Py_LeaveRecursiveCallTstate(tstate); return NULL; @@ -2384,10 +2198,13 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, } static void -call_exc_trace(Py_tracefunc func, PyObject *self, - PyThreadState *tstate, - _PyInterpreterFrame *f) +monitor_raise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_RAISE].tools == 0) { + return; + } PyObject *type, *value, *traceback, *orig_traceback, *arg; int err; _PyErr_Fetch(tstate, &type, &value, &orig_traceback); @@ -2401,7 +2218,7 @@ call_exc_trace(Py_tracefunc func, PyObject *self, _PyErr_Restore(tstate, type, value, orig_traceback); return; } - err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, arg); Py_DECREF(arg); if (err == 0) { _PyErr_Restore(tstate, type, value, orig_traceback); @@ -2413,80 +2230,44 @@ call_exc_trace(Py_tracefunc func, PyObject *self, } } -static int -call_trace_protected(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - PyObject *type, *value, *traceback; - int err; - _PyErr_Fetch(tstate, &type, &value, &traceback); - err = call_trace(func, obj, tstate, frame, what, arg); - if (err == 0) - { - _PyErr_Restore(tstate, type, value, traceback); - return 0; - } - else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); - return -1; + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_PY_UNWIND].tools == 0) { + return; } + _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); } + static void -initialize_trace_info(PyTraceInfo *trace_info, _PyInterpreterFrame *frame) +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { - PyCodeObject *code = frame->f_code; - if (trace_info->code != code) { - trace_info->code = code; - _PyCode_InitAddressRange(code, &trace_info->bounds); + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_EXCEPTION_HANDLED].tools == 0) { + return; } + _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); } void PyThreadState_EnterTracing(PyThreadState *tstate) { tstate->tracing++; - tstate->cframe->use_tracing = 0; } void PyThreadState_LeaveTracing(PyThreadState *tstate) { - assert(tstate->tracing > 0 && tstate->cframe->use_tracing == 0); + assert(tstate->tracing > 0); tstate->tracing--; _PyThreadState_UpdateTracingState(tstate); } -static int -call_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) -{ - int result; - if (tstate->tracing) { - return 0; - } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - int old_what = tstate->tracing_what; - tstate->tracing_what = what; - PyThreadState_EnterTracing(tstate); - assert(_PyInterpreterFrame_LASTI(frame) >= 0); - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - f->f_lineno = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - result = func(obj, f, what, arg); - f->f_lineno = 0; - PyThreadState_LeaveTracing(tstate); - tstate->tracing_what = old_what; - return result; -} PyObject* _PyEval_CallTracing(PyObject *func, PyObject *args) @@ -2494,7 +2275,6 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Save and disable tracing PyThreadState *tstate = _PyThreadState_GET(); int save_tracing = tstate->tracing; - int save_use_tracing = tstate->cframe->use_tracing; tstate->tracing = 0; // Call the tracing function @@ -2502,81 +2282,9 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Restore tracing tstate->tracing = save_tracing; - tstate->cframe->use_tracing = save_use_tracing; - return result; -} - -/* See Objects/lnotab_notes.txt for a description of how tracing works. */ -static int -maybe_call_line_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, int instr_prev) -{ - int result = 0; - - /* If the last instruction falls at the start of a line or if it - represents a jump backwards, update the frame's line number and - then call the trace function if we're tracing source lines. - */ - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - int lastline; - if (instr_prev <= frame->f_code->_co_firsttraceable) { - lastline = -1; - } - else { - lastline = _PyCode_LineNumberFromArray(frame->f_code, instr_prev); - } - int line = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - if (line != -1 && f->f_trace_lines) { - /* Trace backward edges (except in 'yield from') or if line number has changed */ - int trace = line != lastline || - (_PyInterpreterFrame_LASTI(frame) < instr_prev && - // SEND has no quickened forms, so no need to use _PyOpcode_Deopt - // here: - _Py_OPCODE(*frame->prev_instr) != SEND); - if (trace) { - result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); - } - } - /* Always emit an opcode event if we're tracing all opcodes. */ - if (f->f_trace_opcodes && result == 0) { - result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None); - } return result; } -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } - - tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; - tstate->c_profileobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new profile function is fully - // set, so it's safe to call sys.setprofile() again (reentrant call). - Py_XDECREF(old_profileobj); - - return 0; -} - void PyEval_SetProfile(Py_tracefunc func, PyObject *arg) { @@ -2608,33 +2316,6 @@ PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) } } -int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { - return -1; - } - - tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; - tstate->c_traceobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new trace function is fully - // set, so it's safe to call sys.settrace() again (reentrant call). - Py_XDECREF(old_traceobj); - - return 0; -} - void PyEval_SetTrace(Py_tracefunc func, PyObject *arg) { @@ -2863,114 +2544,6 @@ PyEval_GetFuncDesc(PyObject *func) return " object"; } -#define C_TRACE(x, call) \ -if (use_tracing && tstate->c_profilefunc) { \ - if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_CALL, func)) { \ - x = NULL; \ - } \ - else { \ - x = call; \ - if (tstate->c_profilefunc != NULL) { \ - if (x == NULL) { \ - call_trace_protected(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_EXCEPTION, func); \ - /* XXX should pass (type, value, tb) */ \ - } else { \ - if (call_trace(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_RETURN, func)) { \ - Py_DECREF(x); \ - x = NULL; \ - } \ - } \ - } \ - } \ -} else { \ - x = call; \ - } - - -static PyObject * -trace_call_function(PyThreadState *tstate, - PyObject *func, - PyObject **args, Py_ssize_t nargs, - PyObject *kwnames) -{ - int use_tracing = 1; - PyObject *x; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames)); - return x; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = args[0]; - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - C_TRACE(x, PyObject_Vectorcall(func, - args+1, nargs-1, - kwnames)); - Py_DECREF(func); - return x; - } - return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); -} - -static PyObject * -do_call_core(PyThreadState *tstate, - PyObject *func, - PyObject *callargs, - PyObject *kwdict, - int use_tracing - ) -{ - PyObject *result; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(result, PyObject_Call(func, callargs, kwdict)); - return result; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) { - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - if (nargs > 0 && use_tracing) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = PyTuple_GET_ITEM(callargs, 0); - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - - C_TRACE(result, _PyObject_FastCallDictTstate( - tstate, func, - &_PyTuple_ITEMS(callargs)[1], - nargs - 1, - kwdict)); - Py_DECREF(func); - return result; - } - } - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - return PyObject_Call(func, callargs, kwdict); -} - /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, @@ -3438,69 +3011,6 @@ _PyEval_RequestCodeExtraIndex(freefunc free) return new_index; } -static void -dtrace_function_entry(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); -} - -static void -dtrace_function_return(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); -} - -/* DTrace equivalent of maybe_call_line_trace. */ -static void -maybe_dtrace_line(_PyInterpreterFrame *frame, - PyTraceInfo *trace_info, - int instr_prev) -{ - const char *co_filename, *co_name; - - /* If the last instruction executed isn't in the current - instruction window, reset the window. - */ - initialize_trace_info(trace_info, frame); - int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds); - int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); - int line = _PyCode_CheckLineNumber(addr, &trace_info->bounds); - if (line != -1) { - /* Trace backward edges or first instruction of a new line */ - if (_PyInterpreterFrame_LASTI(frame) < instr_prev || - (line != lastline && addr == trace_info->bounds.ar_start)) - { - co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); - if (!co_filename) { - co_filename = "?"; - } - co_name = PyUnicode_AsUTF8(frame->f_code->co_name); - if (!co_name) { - co_name = "?"; - } - PyDTrace_LINE(co_filename, co_name, line); - } - } -} - /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions for the limited API. */ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h new file mode 100644 index 00000000000000..45060157416114 --- /dev/null +++ b/Python/clinic/instrumentation.c.h @@ -0,0 +1,196 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(monitoring_use_tool__doc__, +"use_tool($module, tool_id, name, /)\n" +"--\n" +"\n"); + +#define MONITORING_USE_TOOL_METHODDEF \ + {"use_tool", _PyCFunction_CAST(monitoring_use_tool), METH_FASTCALL, monitoring_use_tool__doc__}, + +static PyObject * +monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name); + +static PyObject * +monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + PyObject *name; + + if (!_PyArg_CheckPositional("use_tool", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + name = args[1]; + return_value = monitoring_use_tool_impl(module, tool_id, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_free_tool__doc__, +"free_tool($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_FREE_TOOL_METHODDEF \ + {"free_tool", (PyCFunction)monitoring_free_tool, METH_O, monitoring_free_tool__doc__}, + +static PyObject * +monitoring_free_tool_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_free_tool(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_free_tool_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_tool__doc__, +"get_tool($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_TOOL_METHODDEF \ + {"get_tool", (PyCFunction)monitoring_get_tool, METH_O, monitoring_get_tool__doc__}, + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_tool(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_tool_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_register_callback__doc__, +"register_callback($module, tool_id, event, func, /)\n" +"--\n" +"\n"); + +#define MONITORING_REGISTER_CALLBACK_METHODDEF \ + {"register_callback", _PyCFunction_CAST(monitoring_register_callback), METH_FASTCALL, monitoring_register_callback__doc__}, + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func); + +static PyObject * +monitoring_register_callback(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event; + PyObject *func; + + if (!_PyArg_CheckPositional("register_callback", nargs, 3, 3)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event = _PyLong_AsInt(args[1]); + if (event == -1 && PyErr_Occurred()) { + goto exit; + } + func = args[2]; + return_value = monitoring_register_callback_impl(module, tool_id, event, func); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_events__doc__, +"get_events($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_EVENTS_METHODDEF \ + {"get_events", (PyCFunction)monitoring_get_events, METH_O, monitoring_get_events__doc__}, + +static PyObject * +monitoring_get_events_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_events(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_events_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_events__doc__, +"set_events($module, tool_id, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_EVENTS_METHODDEF \ + {"set_events", _PyCFunction_CAST(monitoring_set_events), METH_FASTCALL, monitoring_set_events__doc__}, + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set); + +static PyObject * +monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event_set; + + if (!_PyArg_CheckPositional("set_events", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event_set = _PyLong_AsInt(args[1]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_events_impl(module, tool_id, event_set); + +exit: + return return_value; +} +/*[clinic end generated code: output=876a03223495a81d input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2ad7cc9e65785a..b9a9fe90a13c4a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5,9 +5,35 @@ DISPATCH(); } + TARGET(INSTRUMENTED_RESUME) { + /* Check monitoring *before* calling instrument */ + /* Possibly combine this with eval breaker */ + assert(oparg < 2); + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + } + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } + DISPATCH(); + } + TARGET(RESUME) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + if (err) goto error; + err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -137,7 +163,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *prod; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -155,7 +180,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *prod; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -175,7 +199,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sub; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -193,7 +216,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sub; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -212,7 +234,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *res; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -229,7 +250,6 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = PEEK(1); PyObject *left = PEEK(2); - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -264,7 +284,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sum; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -284,7 +303,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sum; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -305,7 +323,6 @@ PyObject *res; _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -366,7 +383,6 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -391,7 +407,6 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *tuple = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -416,7 +431,6 @@ } TARGET(BINARY_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *dict = SECOND(); DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -498,7 +512,6 @@ PyObject *v = PEEK(3); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -517,7 +530,6 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); PyObject *value = THIRD(); @@ -543,7 +555,6 @@ } TARGET(STORE_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *dict = SECOND(); PyObject *value = THIRD(); @@ -620,19 +631,26 @@ assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; } + TARGET(INSTRUMENTED_RETURN_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, val); + if (err) goto error; + GO_TO_INSTRUCTION(RETURN_VALUE); + } + TARGET(RETURN_VALUE) { + PREDICTED(RETURN_VALUE); PyObject *retval = POP(); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); frame = cframe.current_frame = pop_frame(tstate, frame); @@ -789,7 +807,7 @@ if (retval == NULL) { if (tstate->c_tracefunc != NULL && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -832,7 +850,17 @@ DISPATCH(); } + TARGET(INSTRUMENTED_YIELD_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, val); + if (err) goto error; + GO_TO_INSTRUCTION(YIELD_VALUE); + } + TARGET(YIELD_VALUE) { + PREDICTED(YIELD_VALUE); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -842,8 +870,6 @@ PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1058,7 +1084,6 @@ PREDICTED(UNPACK_SEQUENCE); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1137,7 +1162,6 @@ PREDICTED(STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1270,7 +1294,6 @@ PREDICTED(LOAD_GLOBAL); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1330,7 +1353,6 @@ } TARGET(LOAD_GLOBAL_MODULE) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; @@ -1350,7 +1372,6 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1719,7 +1740,6 @@ PREDICTED(LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1780,7 +1800,6 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1805,7 +1824,6 @@ } TARGET(LOAD_ATTR_MODULE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1830,7 +1848,6 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1869,7 +1886,6 @@ } TARGET(LOAD_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1891,7 +1907,6 @@ } TARGET(LOAD_ATTR_CLASS) { - assert(cframe.use_tracing == 0); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *cls = TOP(); @@ -1914,7 +1929,6 @@ } TARGET(LOAD_ATTR_PROPERTY) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -1952,7 +1966,6 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *owner = TOP(); @@ -1992,7 +2005,6 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2021,7 +2033,6 @@ } TARGET(STORE_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2073,7 +2084,6 @@ } TARGET(STORE_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2112,7 +2122,6 @@ PREDICTED(COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2125,7 +2134,6 @@ } TARGET(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2156,7 +2164,6 @@ } TARGET(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2188,7 +2195,6 @@ } TARGET(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int invert = cache->mask; @@ -2604,7 +2610,6 @@ PREDICTED(FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2624,7 +2629,7 @@ goto error; } else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); } _PyErr_Clear(tstate); } @@ -2639,7 +2644,6 @@ } TARGET(FOR_ITER_LIST) { - assert(cframe.use_tracing == 0); _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2662,7 +2666,6 @@ } TARGET(FOR_ITER_RANGE) { - assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2686,7 +2689,6 @@ } TARGET(FOR_ITER_GEN) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)TOP(); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2826,7 +2828,6 @@ TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { /* Cached method object */ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2851,7 +2852,6 @@ TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2877,7 +2877,6 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2895,7 +2894,6 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2936,11 +2934,23 @@ DISPATCH(); } + TARGET(INSTRUMENTED_CALL) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function); + if (err) goto error; + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + } + TARGET(CALL) { PREDICTED(CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); @@ -2993,16 +3003,26 @@ } /* Callable is not a normal Python function */ PyObject *res; - if (cframe.use_tracing) { - res = trace_call_function( - tstate, function, stack_pointer-total_args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - function, stack_pointer-total_args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + if (!PyFunction_Check(function) && !PyMethod_Check(function)) { + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); + } + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3096,7 +3116,6 @@ TARGET(CALL_NO_KW_TYPE_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *obj = TOP(); @@ -3114,7 +3133,6 @@ TARGET(CALL_NO_KW_STR_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3184,7 +3202,6 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { - assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3218,7 +3235,6 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3258,7 +3274,6 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3297,7 +3312,6 @@ } TARGET(CALL_NO_KW_LEN) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); @@ -3327,7 +3341,6 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); @@ -3360,7 +3373,6 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); @@ -3524,6 +3536,10 @@ DISPATCH(); } + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); PyObject *func, *callargs, *kwargs = NULL, *result; @@ -3546,8 +3562,30 @@ } } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func); + if (err) goto error; + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); @@ -3723,7 +3761,6 @@ PREDICTED(BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3745,7 +3782,6 @@ TARGET(EXTENDED_ARG) { assert(oparg); - assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c new file mode 100644 index 00000000000000..235cfce6c44635 --- /dev/null +++ b/Python/instrumentation.c @@ -0,0 +1,573 @@ + + +#include "Python.h" +#include "pycore_frame.h" +#include "pycore_interp.h" +#include "pycore_namespace.h" +#include "pycore_opcode.h" +#include "pycore_pyerrors.h" + + +static PyObject DISABLE = +{ + 1 << 30, + &PyBaseObject_Type +}; + +static int +call_multiple_instruments( + PyInterpreterState *interp, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargs, _Py_CODEUNIT *instr) +{ + PyErr_SetString(PyExc_SystemError, "Multiple tools not supported yet"); + return -1; +} + +static const uint8_t DE_INSTRUMENT[256] = { + [INSTRUMENTED_RESUME] = RESUME, + [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, + [INSTRUMENTED_CALL] = CALL, + [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, +/* + [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, +*/ +}; + +static int +call_instrument( + PyThreadState *tstate, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) +{ + assert(!_PyErr_Occurred(tstate)); + PyInterpreterState *interp = tstate->interp; + int sole_tool_plus1 = interp->sole_tool_plus1[event]; + if (sole_tool_plus1 <= 0) { + if (sole_tool_plus1 == 0) { + /* Why is this instrumented if there is no tool? */ + return 0; + } + return call_multiple_instruments(interp, code, event, args, nargsf, instr); + } + PyObject *instrument = interp->tools[sole_tool_plus1-1].instrument_callables[event]; + if (instrument == NULL) { + return 0; + } + PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + if (res == NULL) { + return -1; + } + if (res == &DISABLE) { + /* Remove this instrument */ + assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); + _Py_SET_OPCODE(*instr, DE_INSTRUMENT[_Py_OPCODE(*instr)]); + } + Py_DECREF(res); + return 0; +} + + +int +_Py_call_instrumentation( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (tstate->tracing) { + return 0; + } + tstate->tracing++; + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; + return err; +} + +int +_Py_call_instrumentation_arg( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) +{ + if (tstate->tracing) { + return 0; + } + tstate->tracing++; + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; + return err; +} + +void +_Py_call_instrumentation_exc( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) +{ + assert(_PyErr_Occurred(tstate)); + if (tstate->tracing) { + return; + } + tstate->tracing++; + PyObject *type, *value, *traceback; + _PyErr_Fetch(tstate, &type, &value, &traceback); + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; + Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); + int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + if (err) { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + } + else { + _PyErr_Restore(tstate, type, value, traceback); + } + assert(_PyErr_Occurred(tstate)); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; +} + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_Get(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); + PyObject *callback = is->tools[tool_id].instrument_callables[event_id]; + is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); + return callback; +} + +static const uint8_t INSTRUMENTED_OPCODES[256] = { + [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + /* + [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + */ +}; + +#define SET1(a) (1 << (a)) +#define SET2(a, b ) ( (1 << (a)) | (1 << (b)) ) +#define SET3(a, b, c) ( (1 << (a)) | (1 << (b)) | (1 << (c)) ) + +#define CALL_EVENTS \ + SET3(PY_MONITORING_EVENT_CALL, PY_MONITORING_EVENT_C_RAISE, PY_MONITORING_EVENT_C_RETURN) + +static const _PyMonitoringEventSet EVENTS_FOR_OPCODE[256] = { + [RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), + [INSTRUMENTED_RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), + [CALL] = CALL_EVENTS, + [INSTRUMENTED_CALL] = CALL_EVENTS, + [CALL_FUNCTION_EX] = CALL_EVENTS, + [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_EVENTS, + [RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), + [INSTRUMENTED_RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), + [YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), + [INSTRUMENTED_YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), +}; + +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + if (interp->monitoring_version == code->_co_instrument_version) { + return 0; + } + /* First establish how much extra space will need, + * and free/allocate it if different to what we had before */ + /* For now only allow only one tool per event */ + assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_LINE].tools == 0); + assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_INSTRUCTION].tools == 0); + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + assert(interp->monitoring_tools_per_event[i].tools == 0 || interp->sole_tool_plus1[i] > 0); + } + //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", + // code, code->_co_instrument_version, interp->monitoring_version); + + /* Insert basic instrumentation */ + int instrumented = 0; + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + if (_PyOpcode_Deopt[opcode]) { + opcode = _PyOpcode_Deopt[opcode]; + } + /* Don't instrument RESUME in await and yield from. */ + if (opcode == RESUME && _Py_OPARG(_PyCode_CODE(code)[i]) >= 2) { + continue; + } + _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; + if (events & interp->monitored_events) { + instrumented++; + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); + } + i += _PyOpcode_Caches[opcode]; + } + //printf("Instrumented %d instructions\n", instrumented); + code->_co_instrument_version = interp->monitoring_version; + return 0; +} + +void +_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (interp->events_per_monitoring_tool[tool_id] == events) { + return; + } + interp->events_per_monitoring_tool[tool_id] = events; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + _PyMonitoringToolSet tools = { 0 }; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + int event_tool_pair = (interp->events_per_monitoring_tool[t] >> e) & 1; + tools.tools |= (event_tool_pair << t); + } + interp->monitoring_tools_per_event[e] = tools; + } + int multitools = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + _PyMonitoringToolSet tools = interp->monitoring_tools_per_event[e]; + if (_Py_popcount32(tools.tools) <= 1) { + interp->sole_tool_plus1[e] = _Py_bit_length(tools.tools); + } + else { + multitools++; + interp->sole_tool_plus1[e] = -1; + } + } + if (multitools) { + /* Only support one tool for now */ + assert(0 && "Only support one tool for now"); + } + interp->monitoring_version++; + + _PyMonitoringEventSet monitored_events = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + monitored_events |= interp->events_per_monitoring_tool[t]; + } + interp->monitored_events = monitored_events; + /* Instrument all executing code objects */ + + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + + while (ts) { + _PyInterpreterFrame *frame = ts->cframe->current_frame; + while (frame) { + if (frame->owner != FRAME_OWNED_BY_CSTACK) { + _Py_Instrument(frame->f_code, interp); + } + frame = frame->previous; + } + ts = PyThreadState_Next(ts); + } +} + +/* List of objects in monitoring namespace + class Event(enum.IntFlag) + def use_tool_id(id)->None + def free_tool_id(id)->None + def get_events(tool_id: int)->Event + def set_events(tool_id: int, event_set: Event)->None + def get_local_events(tool_id: int, code: CodeType)->Event + def set_local_events(tool_id: int, code: CodeType, event_set: Event)->None + def register_callback(tool_id: int, event: Event, func: Callable)->Optional[Callable] + def insert_marker(tool_id: int, code: CodeType, offset: Event)->None + def remove_marker(tool_id: int, code: CodeType, offset: Event)->None + def restart_events()->None + DISABLE: object +*/ + +/*[clinic input] +module monitoring +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/ +/*[clinic end generated code]*/ + +#include "clinic/instrumentation.c.h" + + +static int +check_valid_tool(int tool_id) +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_Format(PyExc_ValueError, "invalid tool ID: %d (must be between 0 and 5)", tool_id); + return -1; + } + return 0; +} + + +/*[clinic input] +monitoring.use_tool + + tool_id: int + name: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) +/*[clinic end generated code: output=d00b74d147bab1e3 input=506e604e1ea75567]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (interp->monitoring_tool_names[tool_id] != NULL) { + PyErr_SetString(PyExc_ValueError, "tool ID is already in use"); + return NULL; + } + interp->monitoring_tool_names[tool_id] = Py_NewRef(name); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.free_tool + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_free_tool_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=7893bfdad26f51fa input=919fecb6f63130ed]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + Py_CLEAR(interp->monitoring_tool_names[tool_id]); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.get_tool + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + PyObject *name = interp->monitoring_tool_names[tool_id]; + if (name == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(name); +} + +/*[clinic input] +monitoring.register_callback + + + tool_id: int + event: int + func: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func) +/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (_Py_popcount32(event) != 1) { + PyErr_SetString(PyExc_ValueError, "The callaback can only be set for one event at a time"); + } + int event_id = _Py_bit_length(event)-1; + if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { + PyErr_Format(PyExc_ValueError, "invalid event: %d", event); + return NULL; + } + if (func == Py_None) { + func = NULL; + } + func = _PyMonitoring_RegisterCallback(tool_id, event_id, func); + if (func == NULL) { + Py_RETURN_NONE; + } + return func; +} + +/*[clinic input] +monitoring.get_events + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_events_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + _PyMonitoringEventSet event_set = interp->events_per_monitoring_tool[tool_id]; + return PyLong_FromUnsignedLong(event_set); +} + +/*[clinic input] +monitoring.set_events + + tool_id: int + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) +/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/ +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + _PyMonitoring_SetEvents(tool_id, event_set); + Py_RETURN_NONE; +} + + +static PyMethodDef methods[] = { + MONITORING_USE_TOOL_METHODDEF + MONITORING_FREE_TOOL_METHODDEF + MONITORING_GET_TOOL_METHODDEF + MONITORING_REGISTER_CALLBACK_METHODDEF + MONITORING_GET_EVENTS_METHODDEF + MONITORING_SET_EVENTS_METHODDEF + {NULL, NULL} // sentinel +}; + +static struct PyModuleDef monitoring_module = { + PyModuleDef_HEAD_INIT, + "sys.monitoring", + NULL, + -1, /* multiple "initialization" just copies the module dict. */ + methods, + NULL, + NULL, + NULL, + NULL +}; + +static int +add_power2_constant(PyObject *obj, const char *name, int i) +{ + PyObject *val = PyLong_FromLong(1< +#include "Python.h" +#include "pycore_instruments.h" +#include "pycore_pyerrors.h" +#include "pycore_pymem.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_sysmodule.h" + +typedef struct _PyLegacyEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + int event; +} _PyLegacyEventHandler; + +static void +dealloc(_PyLegacyEventHandler *self) +{ + PyObject_Free(self); +} + + +/* The Py_tracefunc function expects the following arguments: + * frame: FrameObject + * kind: c-int + * arg: The arg + */ + +static PyObject * +call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_profilefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject* frame = PyEval_GetFrame(); + Py_INCREF(frame); + assert(frame != NULL); + int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +sys_profile_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + return call_profile_func(self, Py_None); +} + +static PyObject * +sys_profile_func3( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_profile_func(self, args[2]); +} + +static PyObject * +sys_profile_call_or_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + PyObject *callable = args[2]; + if (PyFunction_Check(callable) || PyMethod_Check(callable)) { + Py_RETURN_NONE; + } + return call_profile_func(self, callable); +} + +static PyObject * +sys_profile_exception_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + _PyErr_StackItem *err_info = _PyErr_GetTopmostException(_PyThreadState_GET()); + PyObject *arg = _PyErr_StackItemToExcInfoTuple(err_info); + if (arg == NULL) { + return NULL; + } + PyObject *res = call_profile_func(self, arg); + Py_DECREF(arg); + return res; +} + +static PyObject * +sys_trace_none_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_profile_func(self, Py_None); +} + + +PyTypeObject _PyLegacyEventHandler_Type = { + + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "sys.profile_event_handler", + sizeof(_PyLegacyEventHandler), + .tp_dealloc = (destructor)dealloc, + .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL, + .tp_call = PyVectorcall_Call, +}; + +static int +set_callbacks(int tool, vectorcallfunc vectorcall, int legacy_event, int event1, int event2) +{ + _PyLegacyEventHandler *callback = + PyObject_NEW(_PyLegacyEventHandler, &_PyLegacyEventHandler_Type); + if (callback == NULL) { + return -1; + } + callback->vectorcall = vectorcall; + callback->event = legacy_event; + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event1, (PyObject *)callback)); + if (event2 >= 0) { + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event2, (PyObject *)callback)); + } + Py_DECREF(callback); + return 0; +} + +#ifndef NDEBUG +/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and + PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen + when a thread continues to run after Python finalization, especially + daemon threads. */ +static int +is_tstate_valid(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + +int +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; + } + int delta = (func != NULL) - (tstate->c_profilefunc != NULL); + tstate->c_profilefunc = func; + PyObject *old_profileobj = tstate->c_profileobj; + tstate->c_profileobj = Py_XNewRef(arg); + /* Flag that tracing or profiling is turned on */ + _PyThreadState_UpdateTracingState(tstate); + + /* Setup PEP 669 monitoring callbacks and events. */ + tstate->interp->sys_profiling_threads += delta; + assert(tstate->interp->sys_profiling_threads >= 0); + if (!tstate->interp->sys_profile_initialized) { + tstate->interp->sys_profile_initialized = true; + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + } + if (tstate->interp->sys_profiling_threads && delta) { + uint32_t events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_C_RETURN) | + (1 << PY_MONITORING_EVENT_C_RAISE); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + } + else if (tstate->interp->sys_profiling_threads == 0) { + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + } + // gh-98257: Only call Py_XDECREF() once the new profile function is fully + // set, so it's safe to call sys.setprofile() again (reentrant call). + Py_XDECREF(old_profileobj); + return 0; +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); + tstate->c_tracefunc = func; + PyObject *old_traceobj = tstate->c_traceobj; + tstate->c_traceobj = Py_XNewRef(arg); + /* Flag that tracing or profiling is turned on */ + _PyThreadState_UpdateTracingState(tstate); + + /* Setup PEP 669 monitoring callbacks and events. */ + tstate->interp->sys_tracing_threads += delta; + assert(tstate->interp->sys_tracing_threads >= 0); + if (!tstate->interp->sys_trace_initialized) { + tstate->interp->sys_trace_initialized = true; + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_none_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + } + if (tstate->interp->sys_tracing_threads) { + uint32_t events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE); + if (tstate->interp->f_opcode_trace_set) { + events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); + } + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + } + else { + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + } + Py_XDECREF(old_traceobj); + return 0; +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 237d3b946b1066..9ec1dd4f09b7cf 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -239,11 +239,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pystate.c b/Python/pystate.c index 1c92ce5d954458..8bbd529d45801a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -309,9 +309,15 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for(int i = 0; i < PY_INSTRUMENT_EVENTS; i++) { - interp->instrumented[i] = 0; - interp->instrument_callables[i] = Py_NewRef(Py_None); + for(int i = 0; i < PY_MONITORING_EVENTS; i++) { + interp->monitoring_tools_per_event[i].tools = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + interp->tools[t].instrument_callables[i] = NULL; + } + } + interp->monitored_events = 0; + for(int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { + interp->events_per_monitoring_tool[i] = 0; } interp->f_opcode_trace_set = false; interp->_initialized = 1; diff --git a/Python/specialize.c b/Python/specialize.c index f84596751f9186..01bf1e7b5cc37f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1707,6 +1707,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5de684e7195bff..7e0f717c7eb9b5 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3369,6 +3369,7 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict) return _PyStatus_ERR("can't set preliminary stderr"); } +PyObject *_Py_CreateMonitoringObject(void); /* Create sys module without all attributes. _PySys_UpdateConfig() should be called later to add remaining attributes. */ @@ -3414,6 +3415,14 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) goto error; } + PyObject *monitoring = _Py_CreateMonitoringObject(); + if (monitoring == NULL) { + goto error; + } + if (PyDict_SetItemString(sysdict, "monitoring", monitoring) < 0) { + goto error; + } + assert(!_PyErr_Occurred(tstate)); *sysmod_p = sysmod; From 4edb7b7ee14cb5ca08e94e4be08765c2b140744b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 28 Nov 2022 15:03:18 +0000 Subject: [PATCH 003/116] Support disabling and restarting. --- Include/cpython/code.h | 1 + Include/cpython/pystate.h | 1 + Include/internal/pycore_interp.h | 1 + Lib/test/test_monitoring.py | 49 ++++++++++++++++- Python/bytecodes.c | 19 +++---- Python/ceval.c | 1 + Python/clinic/instrumentation.c.h | 19 ++++++- Python/generated_cases.c.h | 19 +++---- Python/instrumentation.c | 87 ++++++++++++++++++++----------- 9 files changed, 146 insertions(+), 51 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index da05f316226e69..fdfa93cd75dd7f 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -99,6 +99,7 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ uint64_t _co_instrument_version; /* current instrumentation version */ \ + uint32_t _co_monitored_events; /* current instrumentation */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 99bc0a89dc0e5f..3515d7e04b1a64 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,6 +133,7 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ + uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1a606a72ae2fd1..eb3a96cd1d460c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -90,6 +90,7 @@ struct _is { PyInterpreterState *next; uint64_t monitoring_version; + uint64_t last_restart_version;; struct pythreads { uint64_t next_unique_id; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 4e3428effafc7f..25e959a2c5ce01 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -23,7 +23,8 @@ def g1(): for _ in gen(): pass -TEST_TOOL = 4 +TEST_TOOL = 3 +TEST_TOOL2 = 4 class MonitoringBaseTest(unittest.TestCase): @@ -40,7 +41,7 @@ def test_has_objects(self): m.register_callback # m.insert_marker # m.remove_marker - # m.restart_events + m.restart_events m.DISABLE def test_tool(self): @@ -269,3 +270,47 @@ def call(code, offset, callable): self.assertEqual(stack, [sys._getframe()]) +class CounterWithDisable: + def __init__(self): + self.disable = False + self.count = 0 + def __call__(self, *args): + self.count += 1 + if self.disable: + return sys.monitoring.DISABLE + + +class MontoringDisableAndRestartTest(unittest.TestCase): + + def test_disable(self): + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + + def test_restart(self): + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + + + diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d0ccb70b40f0da..1bd36f99185328 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -116,12 +116,15 @@ dummy_func( if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } + next_instr--; } - int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - ERROR_IF(err, error); - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { - goto handle_eval_breaker; + else { + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } } } @@ -132,11 +135,9 @@ dummy_func( if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); - err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - ERROR_IF(err, error); + next_instr--; } - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/ceval.c b/Python/ceval.c index b3695ac7af1d98..ce061ee3d52679 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2257,6 +2257,7 @@ monitor_handled(PyThreadState *tstate, void PyThreadState_EnterTracing(PyThreadState *tstate) { + assert(tstate->tracing >= 0); tstate->tracing++; } diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 45060157416114..e677ea86d3647d 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -193,4 +193,21 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=876a03223495a81d input=a9049054013a1b77]*/ + +PyDoc_STRVAR(monitoring_restart_events__doc__, +"restart_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING_RESTART_EVENTS_METHODDEF \ + {"restart_events", (PyCFunction)monitoring_restart_events, METH_NOARGS, monitoring_restart_events__doc__}, + +static PyObject * +monitoring_restart_events_impl(PyObject *module); + +static PyObject * +monitoring_restart_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring_restart_events_impl(module); +} +/*[clinic end generated code: output=3997247efd06367f input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b9a9fe90a13c4a..6f6e20f06c4456 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13,12 +13,15 @@ if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } + next_instr--; } - int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - if (err) goto error; - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { - goto handle_eval_breaker; + else { + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } } DISPATCH(); } @@ -30,11 +33,9 @@ if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; - err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - if (err) goto error; + next_instr--; } - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 235cfce6c44635..a9fd50905adffb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -24,9 +24,11 @@ call_multiple_instruments( } static const uint8_t DE_INSTRUMENT[256] = { + [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, [INSTRUMENTED_CALL] = CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, /* [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, @@ -55,18 +57,24 @@ call_instrument( } return call_multiple_instruments(interp, code, event, args, nargsf, instr); } - PyObject *instrument = interp->tools[sole_tool_plus1-1].instrument_callables[event]; + int sole_tool = sole_tool_plus1-1; + if (tstate->monitoring & (1 << sole_tool)) { + return 0; + } + PyObject *instrument = interp->tools[sole_tool].instrument_callables[event]; if (instrument == NULL) { return 0; } + tstate->monitoring |= (1 << sole_tool); PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + tstate->monitoring &= ~(1 << sole_tool); if (res == NULL) { return -1; } if (res == &DISABLE) { /* Remove this instrument */ assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); - _Py_SET_OPCODE(*instr, DE_INSTRUMENT[_Py_OPCODE(*instr)]); + *instr = _Py_MAKECODEUNIT(DE_INSTRUMENT[_Py_OPCODE(*instr)], _Py_OPARG(*instr)); } Py_DECREF(res); return 0; @@ -78,17 +86,12 @@ _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->tracing) { - return 0; - } - tstate->tracing++; PyCodeObject *code = frame->f_code; int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); - tstate->tracing--; return err; } @@ -97,17 +100,12 @@ _Py_call_instrumentation_arg( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { - if (tstate->tracing) { - return 0; - } - tstate->tracing++; PyCodeObject *code = frame->f_code; int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); - tstate->tracing--; return err; } @@ -117,10 +115,6 @@ _Py_call_instrumentation_exc( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { assert(_PyErr_Occurred(tstate)); - if (tstate->tracing) { - return; - } - tstate->tracing++; PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; @@ -139,7 +133,6 @@ _Py_call_instrumentation_exc( } assert(_PyErr_Occurred(tstate)); Py_DECREF(instruction_offset_obj); - tstate->tracing--; } PyObject * @@ -212,9 +205,22 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); - + /* Avoid instrumenting code that has been disabled. + * Only instrument new events + */ + _PyMonitoringEventSet new_events = interp->monitored_events & ~code->_co_monitored_events; + _PyMonitoringEventSet removed_events = code->_co_monitored_events & ~interp->monitored_events; + if (interp->last_restart_version > code->_co_instrument_version) { + new_events = interp->monitored_events; + removed_events = 0; + } + code->_co_monitored_events = interp->monitored_events; + code->_co_instrument_version = interp->monitoring_version; + assert((new_events & removed_events) == 0); + if ((new_events | removed_events) == 0) { + return 0; + } /* Insert basic instrumentation */ - int instrumented = 0; int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; @@ -227,15 +233,18 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) continue; } _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; - if (events & interp->monitored_events) { - instrumented++; + if (new_events & events) { assert(INSTRUMENTED_OPCODES[opcode] != 0); *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); } + if (removed_events & events) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } i += _PyOpcode_Caches[opcode]; } - //printf("Instrumented %d instructions\n", instrumented); - code->_co_instrument_version = interp->monitoring_version; return 0; } @@ -321,7 +330,7 @@ static int check_valid_tool(int tool_id) { if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_Format(PyExc_ValueError, "invalid tool ID: %d (must be between 0 and 5)", tool_id); + PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id); return -1; } return 0; @@ -351,7 +360,7 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) } PyInterpreterState *interp = _PyInterpreterState_Get(); if (interp->monitoring_tool_names[tool_id] != NULL) { - PyErr_SetString(PyExc_ValueError, "tool ID is already in use"); + PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id); return NULL; } interp->monitoring_tool_names[tool_id] = Py_NewRef(name); @@ -429,7 +438,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { - PyErr_Format(PyExc_ValueError, "invalid event: %d", event); + PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } if (func == Py_None) { @@ -454,8 +463,7 @@ static PyObject * monitoring_get_events_impl(PyObject *module, int tool_id) /*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + if (check_valid_tool(tool_id)) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); @@ -476,8 +484,7 @@ static PyObject * monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) /*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/ { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + if (check_valid_tool(tool_id)) { return NULL; } if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { @@ -488,6 +495,25 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) Py_RETURN_NONE; } +/*[clinic input] +monitoring.restart_events + +[clinic start generated code]*/ + +static PyObject * +monitoring_restart_events_impl(PyObject *module) +/*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/ +{ + /* We want to ensure that: + * last restart version < current version + * last restart version > instrumented version for all code objects + */ + PyInterpreterState *interp = _PyInterpreterState_Get(); + interp->last_restart_version = interp->monitoring_version + 1; + interp->monitoring_version += 2; + Py_RETURN_NONE; +} + static PyMethodDef methods[] = { MONITORING_USE_TOOL_METHODDEF @@ -496,6 +522,7 @@ static PyMethodDef methods[] = { MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF MONITORING_SET_EVENTS_METHODDEF + MONITORING_RESTART_EVENTS_METHODDEF {NULL, NULL} // sentinel }; From 852c40b6757fecc8d9997734b698812d0cf95536 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 28 Nov 2022 18:27:15 +0000 Subject: [PATCH 004/116] Support multiple tools per event. --- Include/cpython/code.h | 12 +- Include/cpython/pystate.h | 1 - Include/internal/pycore_instruments.h | 25 +- Include/internal/pycore_interp.h | 6 +- Lib/test/test_monitoring.py | 36 ++- Objects/codeobject.c | 7 +- Python/bytecodes.c | 5 +- Python/ceval.c | 6 +- Python/generated_cases.c.h | 5 +- Python/instrumentation.c | 400 +++++++++++++++++++------- Python/pystate.c | 7 +- 11 files changed, 372 insertions(+), 138 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fdfa93cd75dd7f..49b2a0e7108665 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -7,6 +7,12 @@ extern "C" { #endif +#define PY_MONITORING_EVENTS 16 + +typedef struct _Py_MonitoringMatrix { + uint8_t tools[PY_MONITORING_EVENTS]; +} _Py_MonitoringMatrix; + /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses @@ -99,7 +105,11 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ uint64_t _co_instrument_version; /* current instrumentation version */ \ - uint32_t _co_monitored_events; /* current instrumentation */ \ + uint8_t _co_monitoring_data_per_instruction; /* Number of bytes per instruction */ \ + uint8_t _co_line_data_offset; /* Offset in data for line data */ \ + uint8_t _co_opcode_data_offset; /* Offset in data for opcode data */ \ + _Py_MonitoringMatrix _co_monitoring_matrix; \ + uint8_t *_co_monitoring_data; /* array of data for monitoring */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3515d7e04b1a64..99bc0a89dc0e5f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,7 +133,6 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ - uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 4c3a1dbcb3a4cb..781aa568d52508 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -6,13 +6,15 @@ #include "pycore_bitutils.h" // _Py_popcount32 #include "pycore_frame.h" +#include "cpython/code.h" + #ifdef __cplusplus extern "C" { #endif #define PY_MONITORING_TOOL_IDS 8 -/* Require dynamic instrumentation */ +/* Require bytecode instrumentation */ #define PY_MONITORING_EVENT_PY_START 0 #define PY_MONITORING_EVENT_PY_RESUME 1 @@ -24,16 +26,21 @@ extern "C" { #define PY_MONITORING_EVENT_JUMP 7 #define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_INSTRUMENTED_EVENTS 9 + +/* Grouped events */ #define PY_MONITORING_EVENT_C_RETURN 9 -#define PY_MONITORING_EVENT_PY_THROW 10 -#define PY_MONITORING_EVENT_RAISE 11 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 -#define PY_MONITORING_EVENT_C_RAISE 13 +#define PY_MONITORING_EVENT_C_RAISE 10 + +/* Exceptional events */ + +#define PY_MONITORING_EVENT_PY_THROW 11 +#define PY_MONITORING_EVENT_RAISE 12 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 #define PY_MONITORING_EVENT_PY_UNWIND 14 /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ -#define PY_MONITORING_EVENTS 15 /* Temporary and internal events */ @@ -42,12 +49,6 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; -typedef uint16_t _PyMonitoringSet; - -typedef struct _pytoolset { - uint8_t tools; -} _PyMonitoringToolSet; - /* Reserved IDs */ #define PY_INSTRUMENT_PEP_523 5 diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index eb3a96cd1d460c..a614931da6e6b3 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -195,11 +195,11 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _PyMonitoringToolSet monitoring_tools_per_event[PY_MONITORING_EVENTS]; - _PyMonitoringEventSet events_per_monitoring_tool[PY_MONITORING_TOOL_IDS]; - _PyMonitoringEventSet monitored_events; + _Py_MonitoringMatrix monitoring_matrix; /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; + bool multiple_tools; + uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 25e959a2c5ce01..0bb3eb94b5c226 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -49,9 +49,13 @@ def test_tool(self): self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") sys.monitoring.free_tool(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) - sys.monitoring.set_events(TEST_TOOL, 19) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 19) + sys.monitoring.set_events(TEST_TOOL, 15) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) sys.monitoring.set_events(TEST_TOOL, 0) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) class MonitoringCountTest(unittest.TestCase): @@ -66,7 +70,10 @@ def __call__(self, *args): counter = Counter() sys.monitoring.register_callback(TEST_TOOL, event, counter) - sys.monitoring.set_events(TEST_TOOL, event) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) self.assertEqual(counter.count, 0) counter.count = 0 func() @@ -101,17 +108,18 @@ def test_c_return_count(self): (E.PY_RESUME, "resume"), (E.PY_RETURN, "return"), (E.PY_YIELD, "yield"), - (E.C_RAISE, "c_raise"), - (E.C_RETURN, "c_return"), (E.JUMP, "jump"), (E.BRANCH, "branch"), (E.RAISE, "raise"), (E.PY_UNWIND, "unwind"), (E.EXCEPTION_HANDLED, "exception_handled"), + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), ] SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + def just_pass(): pass @@ -313,4 +321,22 @@ def test_restart(self): self.assertEqual(counter.count, 1) +class MultipleMonitors(unittest.TestCase): + + def test_two_simple(self): + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual(count1, 1) + self.assertEqual(count2, 1) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 7657b1322df401..a6a8dd2e475051 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -338,8 +338,11 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->_co_instrument_version = 0; - + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + co->_co_monitoring_matrix.tools[e] = 0; + } + co->_co_monitoring_data = 0; + co->_co_monitoring_data_per_instruction = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1bd36f99185328..64b4e88664c320 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -111,7 +111,6 @@ dummy_func( inst(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - assert(oparg < 2); if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -120,9 +119,9 @@ dummy_func( } else { int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); + tstate, oparg != 0, frame, next_instr-1); ERROR_IF(err, error); - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/ceval.c b/Python/ceval.c index ce061ee3d52679..dee1763ae6af7d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2202,7 +2202,7 @@ monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_RAISE].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } PyObject *type, *value, *traceback, *orig_traceback, *arg; @@ -2236,7 +2236,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_PY_UNWIND].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { return; } _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); @@ -2248,7 +2248,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_EXCEPTION_HANDLED].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_EXCEPTION_HANDLED] == 0) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6f6e20f06c4456..f8141a93813678 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,7 +8,6 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - assert(oparg < 2); if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -17,9 +16,9 @@ } else { int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); + tstate, oparg != 0, frame, next_instr-1); if (err) goto error; - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a9fd50905adffb..55f4c63901c7fa 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,15 +14,6 @@ static PyObject DISABLE = &PyBaseObject_Type }; -static int -call_multiple_instruments( - PyInterpreterState *interp, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargs, _Py_CODEUNIT *instr) -{ - PyErr_SetString(PyExc_SystemError, "Multiple tools not supported yet"); - return -1; -} - static const uint8_t DE_INSTRUMENT[256] = { [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, @@ -42,10 +33,72 @@ static const uint8_t DE_INSTRUMENT[256] = { */ }; +/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ +static int +call_one_instrument( + PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, + Py_ssize_t nargsf, int8_t tool, int event) +{ + if (tstate->tracing) { + return 0; + } + PyObject *instrument = interp->tools[tool].instrument_callables[event]; + if (instrument == NULL) { + return 0; + } + tstate->tracing++; + PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + tstate->tracing--; + if (res == NULL) { + return -1; + } + Py_DECREF(res); + return (res == &DISABLE); +} + +static int +call_multiple_instruments( + PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) +{ + int offset = ((_Py_CODEUNIT *)opcode) - _PyCode_CODE(code); + uint8_t *toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset*code->_co_monitoring_data_per_instruction]; + uint8_t tools = *toolsptr; + /* No line or instruction yet */ + assert(code->_co_monitoring_data_per_instruction == 1); + assert(_Py_popcount32(tools) > 0); + assert(interp->sole_tool_plus1[event] < 0); + /* To do -- speed this up */ + assert(tools != 0); + int tool = 0; + do { + assert(tool < 8); + if (tools & (1 << tool)) { + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); + if (res) { + if (res > 0) { + *toolsptr &= ~(1 << tool); + if (*toolsptr == 0) { + /* Remove this instrument */ + assert(DE_INSTRUMENT[*opcode] != 0); + *opcode = DE_INSTRUMENT[*opcode]; + } + } + else { + return -1; + } + } + } + tool++; + } while (tools); + return 0; +} + static int call_instrument( PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) + PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) { assert(!_PyErr_Occurred(tstate)); PyInterpreterState *interp = tstate->interp; @@ -55,29 +108,17 @@ call_instrument( /* Why is this instrumented if there is no tool? */ return 0; } - return call_multiple_instruments(interp, code, event, args, nargsf, instr); + return call_multiple_instruments(interp, tstate, code, event, args, nargsf, opcode); } int sole_tool = sole_tool_plus1-1; - if (tstate->monitoring & (1 << sole_tool)) { - return 0; - } - PyObject *instrument = interp->tools[sole_tool].instrument_callables[event]; - if (instrument == NULL) { - return 0; - } - tstate->monitoring |= (1 << sole_tool); - PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - tstate->monitoring &= ~(1 << sole_tool); - if (res == NULL) { - return -1; - } - if (res == &DISABLE) { + int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); + if (res > 0) { /* Remove this instrument */ - assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); - *instr = _Py_MAKECODEUNIT(DE_INSTRUMENT[_Py_OPCODE(*instr)], _Py_OPARG(*instr)); + assert(DE_INSTRUMENT[*opcode] != 0); + *opcode = DE_INSTRUMENT[*opcode]; + res = 0; } - Py_DECREF(res); - return 0; + return res; } @@ -90,7 +131,7 @@ _Py_call_instrumentation( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); Py_DECREF(instruction_offset_obj); return err; } @@ -104,7 +145,7 @@ _Py_call_instrumentation_arg( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); Py_DECREF(instruction_offset_obj); return err; } @@ -122,7 +163,7 @@ _Py_call_instrumentation_exc( PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + int err = call_instrument(tstate, code, event, &args[1], nargsf, (uint8_t*)instr); if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -145,13 +186,18 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); return callback; } +#define RESUME_0 256 +#define RESUME_1 257 + -static const uint8_t INSTRUMENTED_OPCODES[256] = { +static const uint8_t INSTRUMENTED_OPCODES[258] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [CALL] = INSTRUMENTED_CALL, [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [RESUME] = INSTRUMENTED_RESUME, + [RESUME_0] = INSTRUMENTED_RESUME, + [RESUME_1] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, @@ -169,26 +215,88 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { */ }; -#define SET1(a) (1 << (a)) -#define SET2(a, b ) ( (1 << (a)) | (1 << (b)) ) -#define SET3(a, b, c) ( (1 << (a)) | (1 << (b)) | (1 << (c)) ) - -#define CALL_EVENTS \ - SET3(PY_MONITORING_EVENT_CALL, PY_MONITORING_EVENT_C_RAISE, PY_MONITORING_EVENT_C_RETURN) - -static const _PyMonitoringEventSet EVENTS_FOR_OPCODE[256] = { - [RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), - [INSTRUMENTED_RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), - [CALL] = CALL_EVENTS, - [INSTRUMENTED_CALL] = CALL_EVENTS, - [CALL_FUNCTION_EX] = CALL_EVENTS, - [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_EVENTS, - [RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), - [INSTRUMENTED_RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), - [YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), - [INSTRUMENTED_YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), + +static const int8_t EVENT_FOR_OPCODE[258] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [RESUME_0] = PY_MONITORING_EVENT_PY_START, + [RESUME_1] = PY_MONITORING_EVENT_PY_RESUME, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, +}; + +static const bool OPCODE_HAS_EVENT[258] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [RESUME_0] = true, + [RESUME_1] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, }; +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(0 <= val && val <= 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -198,49 +306,86 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) /* First establish how much extra space will need, * and free/allocate it if different to what we had before */ /* For now only allow only one tool per event */ - assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_LINE].tools == 0); - assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_INSTRUCTION].tools == 0); - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - assert(interp->monitoring_tools_per_event[i].tools == 0 || interp->sole_tool_plus1[i] > 0); + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == 0); + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); + int code_len = (int)Py_SIZE(code); + if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { + /* Don't resize if more space than needed, to avoid churn */ + PyMem_Free(code->_co_monitoring_data); + code->_co_monitoring_data = PyMem_Malloc(code_len * interp->required_monitoring_bytes); + if (code->_co_monitoring_data == NULL) { + code->_co_monitoring_data_per_instruction = 0; + PyErr_NoMemory(); + return -1; + } + code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); /* Avoid instrumenting code that has been disabled. * Only instrument new events */ - _PyMonitoringEventSet new_events = interp->monitored_events & ~code->_co_monitored_events; - _PyMonitoringEventSet removed_events = code->_co_monitored_events & ~interp->monitored_events; + _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); + _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); if (interp->last_restart_version > code->_co_instrument_version) { - new_events = interp->monitored_events; - removed_events = 0; + new_events = interp->monitoring_matrix; + removed_events = (_Py_MonitoringMatrix){ 0 }; } - code->_co_monitored_events = interp->monitored_events; code->_co_instrument_version = interp->monitoring_version; - assert((new_events & removed_events) == 0); - if ((new_events | removed_events) == 0) { + assert(matrix_empty(matrix_and(new_events, removed_events))); + if (matrix_empty(new_events) && matrix_empty(removed_events)) { return 0; } - /* Insert basic instrumentation */ - int code_len = (int)Py_SIZE(code); + /* Insert instrumentation */ for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + int oparg = _Py_OPARG(*instr); if (_PyOpcode_Deopt[opcode]) { opcode = _PyOpcode_Deopt[opcode]; } - /* Don't instrument RESUME in await and yield from. */ - if (opcode == RESUME && _Py_OPARG(_PyCode_CODE(code)[i]) >= 2) { - continue; - } - _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; - if (new_events & events) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); + if (opcode == RESUME) { + if (oparg == 0) { + opcode = RESUME_0; + } + else { + opcode = RESUME_1; + } } - if (removed_events & events) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); + bool has_event = OPCODE_HAS_EVENT[opcode]; + if (has_event) { + int8_t event = EVENT_FOR_OPCODE[opcode]; + assert(event >= 0); + if (interp->sole_tool_plus1[event] >= 0) { + int sole_tool = interp->sole_tool_plus1[event] - 1; + if (new_events.tools[event] & (1 << sole_tool)) { + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); + } + else if (removed_events.tools[event] & (1 << sole_tool)) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } + } + else { + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); + code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] |= new_tools; + } + uint8_t removed_tools = removed_events.tools[event]; + if (removed_tools) { + code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] &= ~removed_tools; + if (code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] == 0) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } + } } } i += _PyOpcode_Caches[opcode]; @@ -248,44 +393,95 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } +#define C_RETURN_EVENTS \ + ((1 << PY_MONITORING_EVENT_C_RETURN) | \ + (1 << PY_MONITORING_EVENT_C_RAISE)) + +#define C_CALL_EVENTS \ + (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) + +static int +cross_check_events(PyInterpreterState *interp) +{ + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + uint8_t tools = interp->monitoring_matrix.tools[e]; + int popcount = _Py_popcount32(tools); + switch(popcount) { + case 0: + if (interp->sole_tool_plus1[e] != 0) { + return 0; + } + break; + case 1: + if (interp->sole_tool_plus1[e] <= 0) { + return 0; + } + if ((tools & (1 << (interp->sole_tool_plus1[e]-1))) == 0) { + return 0; + } + break; + default: + if (interp->sole_tool_plus1[e] >= 0) { + return 0; + } + } + } + return 1; +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { PyInterpreterState *interp = _PyInterpreterState_Get(); - if (interp->events_per_monitoring_tool[tool_id] == events) { + assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); + uint32_t existing_events = matrix_get_events(&interp->monitoring_matrix, tool_id); + if (existing_events == events) { return; } - interp->events_per_monitoring_tool[tool_id] = events; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - _PyMonitoringToolSet tools = { 0 }; - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - int event_tool_pair = (interp->events_per_monitoring_tool[t] >> e) & 1; - tools.tools |= (event_tool_pair << t); - } - interp->monitoring_tools_per_event[e] = tools; - } int multitools = 0; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - _PyMonitoringToolSet tools = interp->monitoring_tools_per_event[e]; - if (_Py_popcount32(tools.tools) <= 1) { - interp->sole_tool_plus1[e] = _Py_bit_length(tools.tools); + int val = (events >> e) & 1; + matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); + uint8_t tools = interp->monitoring_matrix.tools[e]; + if (_Py_popcount32(tools) <= 1) { + interp->sole_tool_plus1[e] = _Py_bit_length(tools); } else { multitools++; interp->sole_tool_plus1[e] = -1; } } + assert(cross_check_events(interp)); + interp->multiple_tools = multitools; if (multitools) { - /* Only support one tool for now */ - assert(0 && "Only support one tool for now"); + interp->multiple_tools = true; + } + bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; + bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; + if (interp->multiple_tools) { + if (opcodes) { + interp->required_monitoring_bytes = 6; + } + else if (lines) { + interp->required_monitoring_bytes = 4; + } + else { + interp->required_monitoring_bytes = 1; + } + } + else { + if (opcodes) { + interp->required_monitoring_bytes = 3; + } + else if (lines) { + interp->required_monitoring_bytes = 2; + } + else { + interp->required_monitoring_bytes = 0; + } } interp->monitoring_version++; - _PyMonitoringEventSet monitored_events = 0; - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - monitored_events |= interp->events_per_monitoring_tool[t]; - } - interp->monitored_events = monitored_events; /* Instrument all executing code objects */ PyThreadState* ts = PyInterpreterState_ThreadHead(interp); @@ -325,7 +521,6 @@ module monitoring #include "clinic/instrumentation.c.h" - static int check_valid_tool(int tool_id) { @@ -336,7 +531,6 @@ check_valid_tool(int tool_id) return 0; } - /*[clinic input] monitoring.use_tool @@ -467,7 +661,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = interp->events_per_monitoring_tool[tool_id]; + _PyMonitoringEventSet event_set = matrix_get_events(&interp->monitoring_matrix, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -491,6 +685,13 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); return NULL; } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + if (event_set & (1 << PY_MONITORING_EVENT_CALL)) { + event_set |= C_RETURN_EVENTS; + } _PyMonitoring_SetEvents(tool_id, event_set); Py_RETURN_NONE; } @@ -566,6 +767,7 @@ const char *event_names[] = { [PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED", [PY_MONITORING_EVENT_C_RAISE] = "C_RAISE", [PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND", + [15] = "Unused", }; PyObject *_Py_CreateMonitoringObject(void) diff --git a/Python/pystate.c b/Python/pystate.c index 8bbd529d45801a..91ad6cdbf2cf27 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -284,7 +284,6 @@ free_interpreter(PyInterpreterState *interp) The runtime state is not manipulated. Instead it is assumed that the interpreter is getting added to the runtime. */ - static void init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, @@ -310,15 +309,11 @@ init_interpreter(PyInterpreterState *interp, PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_tools_per_event[i].tools = 0; + interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; } } - interp->monitored_events = 0; - for(int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { - interp->events_per_monitoring_tool[i] = 0; - } interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 416b3149a935a19ac1a0a73e8ac98f99a156c608 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Dec 2022 14:12:53 +0000 Subject: [PATCH 005/116] Tidy up of monitoring internals --- Include/cpython/pystate.h | 1 + Include/internal/pycore_interp.h | 3 - Lib/test/test_monitoring.py | 31 ++- Python/clinic/instrumentation.c.h | 19 +- Python/instrumentation.c | 417 ++++++++++++++++-------------- Python/legacy_tracing.c | 6 +- Python/pystate.c | 1 + 7 files changed, 284 insertions(+), 194 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 99bc0a89dc0e5f..3515d7e04b1a64 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,6 +133,7 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ + uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a614931da6e6b3..a5c591dde962ef 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -196,9 +196,6 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; - /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ - int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; - bool multiple_tools; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0bb3eb94b5c226..54596148b15913 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -201,6 +201,7 @@ def record_call(code, offset, obj): sys.monitoring.set_events(TEST_TOOL, 0) #Remove the final event, the call to `sys.monitoring.set_events` events = events[:-1] + sys.monitoring.set_events(TEST_TOOL, 0) return events def check_events(self, func, expected=None): @@ -287,7 +288,6 @@ def __call__(self, *args): if self.disable: return sys.monitoring.DISABLE - class MontoringDisableAndRestartTest(unittest.TestCase): def test_disable(self): @@ -305,6 +305,7 @@ def test_disable(self): counter.count = 0 f1() self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) def test_restart(self): counter = CounterWithDisable() @@ -319,11 +320,12 @@ def test_restart(self): counter.count = 0 f1() self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) class MultipleMonitors(unittest.TestCase): - def test_two_simple(self): + def test_two_same(self): counter1 = CounterWithDisable() counter2 = CounterWithDisable() sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) @@ -339,4 +341,29 @@ def test_two_simple(self): count2 = counter2.count self.assertEqual(count1, 1) self.assertEqual(count2, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + + def test_two_different(self): + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual(count1, 1) + self.assertEqual(count2, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index e677ea86d3647d..feecf7695a2a2a 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -210,4 +210,21 @@ monitoring_restart_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring_restart_events_impl(module); } -/*[clinic end generated code: output=3997247efd06367f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(monitoring__all_events__doc__, +"_all_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING__ALL_EVENTS_METHODDEF \ + {"_all_events", (PyCFunction)monitoring__all_events, METH_NOARGS, monitoring__all_events__doc__}, + +static PyObject * +monitoring__all_events_impl(PyObject *module); + +static PyObject * +monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring__all_events_impl(module); +} +/*[clinic end generated code: output=d4c412a002392e2b input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 55f4c63901c7fa..c7cf44efd4d38d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -15,7 +15,6 @@ static PyObject DISABLE = }; static const uint8_t DE_INSTRUMENT[256] = { - [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, [INSTRUMENTED_CALL] = CALL, @@ -33,22 +32,63 @@ static const uint8_t DE_INSTRUMENT[256] = { */ }; + +static const uint8_t INSTRUMENTED_OPCODES[256] = { + [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + /* + [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + */ +}; + +static void +de_instrument(_Py_CODEUNIT *instr) +{ + int opcode = _Py_OPCODE(*instr); + assert(INSTRUMENTED_OPCODES[opcode] == opcode); + int base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[base_opcode]) { + instr[1] = adaptive_counter_warmup(); + } +} + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, Py_ssize_t nargsf, int8_t tool, int event) { - if (tstate->tracing) { + assert(0 <= tool && tool < 8); + int tool_bit = 1 << tool; + if (tstate->monitoring & tool_bit) { return 0; } PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; } - tstate->tracing++; + uint8_t old_monitoring = tstate->monitoring; + tstate->monitoring |= tool_bit; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - tstate->tracing--; + assert((tstate->monitoring & ~tool_bit) == old_monitoring); + tstate->monitoring = old_monitoring; if (res == NULL) { return -1; } @@ -56,72 +96,93 @@ call_one_instrument( return (res == &DISABLE); } +static const int8_t LEFT_MOST_BITS[16] = { + -1, 0, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, +}; + +/* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, + and can perform relatively poorly on platforms without the necessary intrinsics. */ +static int most_significant_bit(uint8_t bits) { + assert(bits != 0); + if (bits > 15) { + return LEFT_MOST_BITS[bits>>4]+4; + } + else { + return LEFT_MOST_BITS[bits]; + } +} + static int call_multiple_instruments( PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { - int offset = ((_Py_CODEUNIT *)opcode) - _PyCode_CODE(code); - uint8_t *toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset*code->_co_monitoring_data_per_instruction]; - uint8_t tools = *toolsptr; - /* No line or instruction yet */ + int offset = instr - _PyCode_CODE(code); + uint8_t *toolsptr; + uint8_t tools; + if (event < PY_MONITORING_EVENT_PY_THROW) { + toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset]; + tools = *toolsptr; + } + else { + toolsptr = NULL; + tools = code->_co_monitoring_matrix.tools[event]; + } + assert((tools & interp->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_monitoring_matrix.tools[event]) == tools); + /* No line or instruction yet -- Remove this assert when adding line/instruction monitoring */ assert(code->_co_monitoring_data_per_instruction == 1); - assert(_Py_popcount32(tools) > 0); - assert(interp->sole_tool_plus1[event] < 0); - /* To do -- speed this up */ - assert(tools != 0); - int tool = 0; - do { + while (tools) { + int tool = most_significant_bit(tools); assert(tool < 8); - if (tools & (1 << tool)) { - tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); - if (res) { - if (res > 0) { - *toolsptr &= ~(1 << tool); - if (*toolsptr == 0) { - /* Remove this instrument */ - assert(DE_INSTRUMENT[*opcode] != 0); - *opcode = DE_INSTRUMENT[*opcode]; - } - } - else { - return -1; - } + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + return -1; + } + else if (event < PY_MONITORING_EVENT_PY_THROW) { + /* DISABLE */ + *toolsptr &= ~(1 << tool); + if (*toolsptr == 0) { + de_instrument(instr); } } - tool++; - } while (tools); + } return 0; } static int call_instrument( PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { assert(!_PyErr_Occurred(tstate)); PyInterpreterState *interp = tstate->interp; - int sole_tool_plus1 = interp->sole_tool_plus1[event]; - if (sole_tool_plus1 <= 0) { - if (sole_tool_plus1 == 0) { - /* Why is this instrumented if there is no tool? */ - return 0; - } - return call_multiple_instruments(interp, tstate, code, event, args, nargsf, opcode); + if (code->_co_monitoring_data) { + return call_multiple_instruments(interp, tstate, code, event, args, nargsf, instr); + } + uint8_t tools = code->_co_monitoring_matrix.tools[event]; + if (tools == 0) { + return 0; } - int sole_tool = sole_tool_plus1-1; + int sole_tool = most_significant_bit(code->_co_monitoring_matrix.tools[event]); + assert(sole_tool >= 0); + assert(_Py_popcount32(code->_co_monitoring_matrix.tools[event]) == 1); int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); if (res > 0) { - /* Remove this instrument */ - assert(DE_INSTRUMENT[*opcode] != 0); - *opcode = DE_INSTRUMENT[*opcode]; + /* DISABLE */ + de_instrument(instr); res = 0; } return res; } - int _Py_call_instrumentation( PyThreadState *tstate, int event, @@ -131,7 +192,7 @@ _Py_call_instrumentation( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); return err; } @@ -145,7 +206,7 @@ _Py_call_instrumentation_arg( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); return err; } @@ -163,7 +224,7 @@ _Py_call_instrumentation_exc( PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -186,37 +247,8 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); return callback; } -#define RESUME_0 256 -#define RESUME_1 257 - -static const uint8_t INSTRUMENTED_OPCODES[258] = { - [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [CALL] = INSTRUMENTED_CALL, - [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [RESUME] = INSTRUMENTED_RESUME, - [RESUME_0] = INSTRUMENTED_RESUME, - [RESUME_1] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - /* - [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, - [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, - [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, - [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - */ -}; - - -static const int8_t EVENT_FOR_OPCODE[258] = { +static const int8_t EVENT_FOR_OPCODE[256] = { [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [CALL] = PY_MONITORING_EVENT_CALL, @@ -224,13 +256,11 @@ static const int8_t EVENT_FOR_OPCODE[258] = { [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [RESUME] = -1, - [RESUME_0] = PY_MONITORING_EVENT_PY_START, - [RESUME_1] = PY_MONITORING_EVENT_PY_RESUME, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, }; -static const bool OPCODE_HAS_EVENT[258] = { +static const bool OPCODE_HAS_EVENT[256] = { [RETURN_VALUE] = true, [INSTRUMENTED_RETURN_VALUE] = true, [CALL] = true, @@ -238,8 +268,7 @@ static const bool OPCODE_HAS_EVENT[258] = { [CALL_FUNCTION_EX] = true, [INSTRUMENTED_CALL_FUNCTION_EX] = true, [RESUME] = true, - [RESUME_0] = true, - [RESUME_1] = true, + [INSTRUMENTED_RESUME] = true, [YIELD_VALUE] = true, [INSTRUMENTED_YIELD_VALUE] = true, }; @@ -311,13 +340,42 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) int code_len = (int)Py_SIZE(code); if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { /* Don't resize if more space than needed, to avoid churn */ - PyMem_Free(code->_co_monitoring_data); - code->_co_monitoring_data = PyMem_Malloc(code_len * interp->required_monitoring_bytes); + code->_co_monitoring_data = PyMem_Realloc(code->_co_monitoring_data, code_len * interp->required_monitoring_bytes); if (code->_co_monitoring_data == NULL) { code->_co_monitoring_data_per_instruction = 0; PyErr_NoMemory(); return -1; } + if (code->_co_monitoring_data_per_instruction == 0) { + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + int base_opcode = _PyOpcode_Deopt[opcode]; + if (OPCODE_HAS_EVENT[base_opcode]) { + bool instrumented = INSTRUMENTED_OPCODES[opcode] == opcode; + if (instrumented) { + int8_t event; + if (base_opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg; + } + else { + event = EVENT_FOR_OPCODE[base_opcode]; + } + assert(event >= 0); + code->_co_monitoring_data[i] = code->_co_monitoring_matrix.tools[event]; + } + else { + code->_co_monitoring_data[i] = 0; + } + } + i += _PyOpcode_Caches[base_opcode]; + } + code->_co_monitoring_data_per_instruction = 1; + } + for (int i = code_len*code->_co_monitoring_data_per_instruction; i < code_len*interp->required_monitoring_bytes; i++) { + code->_co_monitoring_data[i] = 0; + } code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", @@ -327,6 +385,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) */ _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); + code->_co_monitoring_matrix = interp->monitoring_matrix; + if (interp->last_restart_version > code->_co_instrument_version) { new_events = interp->monitoring_matrix; removed_events = (_Py_MonitoringMatrix){ 0 }; @@ -341,54 +401,48 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); int oparg = _Py_OPARG(*instr); - if (_PyOpcode_Deopt[opcode]) { - opcode = _PyOpcode_Deopt[opcode]; + int base_opcode; + if (DE_INSTRUMENT[opcode]) { + base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; } - if (opcode == RESUME) { - if (oparg == 0) { - opcode = RESUME_0; + else { + base_opcode = _PyOpcode_Deopt[opcode]; + } + if (OPCODE_HAS_EVENT[base_opcode]) { + int8_t event; + if (base_opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg; } else { - opcode = RESUME_1; + event = EVENT_FOR_OPCODE[base_opcode]; + assert(event > 0); } - } - bool has_event = OPCODE_HAS_EVENT[opcode]; - if (has_event) { - int8_t event = EVENT_FOR_OPCODE[opcode]; - assert(event >= 0); - if (interp->sole_tool_plus1[event] >= 0) { - int sole_tool = interp->sole_tool_plus1[event] - 1; - if (new_events.tools[event] & (1 << sole_tool)) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); - } - else if (removed_events.tools[event] & (1 << sole_tool)) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); - } + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + assert(INSTRUMENTED_OPCODES[base_opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[base_opcode], oparg); + if (code->_co_monitoring_data) { + code->_co_monitoring_data[i] |= new_tools; } } - else { - uint8_t new_tools = new_events.tools[event]; - if (new_tools) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); - code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] |= new_tools; - } - uint8_t removed_tools = removed_events.tools[event]; - if (removed_tools) { - code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] &= ~removed_tools; - if (code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] == 0) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); - } + uint8_t removed_tools = removed_events.tools[event]; + if (removed_tools) { + if (code->_co_monitoring_data) { + code->_co_monitoring_data[i] &= ~removed_tools; + if (code->_co_monitoring_data[i] == 0) { + de_instrument(instr); } } + else if (INSTRUMENTED_OPCODES[opcode] == opcode) { + /* Cannot remove anything but the only tool */ + assert(code->_co_monitoring_matrix.tools[event] == 0); + assert(_Py_popcount32(removed_tools) == 1); + de_instrument(instr); + } } } - i += _PyOpcode_Caches[opcode]; + i += _PyOpcode_Caches[base_opcode]; } return 0; } @@ -400,35 +454,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) #define C_CALL_EVENTS \ (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) -static int -cross_check_events(PyInterpreterState *interp) -{ - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools = interp->monitoring_matrix.tools[e]; - int popcount = _Py_popcount32(tools); - switch(popcount) { - case 0: - if (interp->sole_tool_plus1[e] != 0) { - return 0; - } - break; - case 1: - if (interp->sole_tool_plus1[e] <= 0) { - return 0; - } - if ((tools & (1 << (interp->sole_tool_plus1[e]-1))) == 0) { - return 0; - } - break; - default: - if (interp->sole_tool_plus1[e] >= 0) { - return 0; - } - } - } - return 1; -} - void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { @@ -443,22 +468,13 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); uint8_t tools = interp->monitoring_matrix.tools[e]; - if (_Py_popcount32(tools) <= 1) { - interp->sole_tool_plus1[e] = _Py_bit_length(tools); - } - else { + if (_Py_popcount32(tools) > 1) { multitools++; - interp->sole_tool_plus1[e] = -1; } } - assert(cross_check_events(interp)); - interp->multiple_tools = multitools; - if (multitools) { - interp->multiple_tools = true; - } bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; - if (interp->multiple_tools) { + if (multitools) { if (opcodes) { interp->required_monitoring_bytes = 6; } @@ -715,30 +731,6 @@ monitoring_restart_events_impl(PyObject *module) Py_RETURN_NONE; } - -static PyMethodDef methods[] = { - MONITORING_USE_TOOL_METHODDEF - MONITORING_FREE_TOOL_METHODDEF - MONITORING_GET_TOOL_METHODDEF - MONITORING_REGISTER_CALLBACK_METHODDEF - MONITORING_GET_EVENTS_METHODDEF - MONITORING_SET_EVENTS_METHODDEF - MONITORING_RESTART_EVENTS_METHODDEF - {NULL, NULL} // sentinel -}; - -static struct PyModuleDef monitoring_module = { - PyModuleDef_HEAD_INIT, - "sys.monitoring", - NULL, - -1, /* multiple "initialization" just copies the module dict. */ - methods, - NULL, - NULL, - NULL, - NULL -}; - static int add_power2_constant(PyObject *obj, const char *name, int i) { @@ -770,6 +762,57 @@ const char *event_names[] = { [15] = "Unused", }; +/*[clinic input] +monitoring._all_events +[clinic start generated code]*/ + +static PyObject * +monitoring__all_events_impl(PyObject *module) +/*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + uint8_t tools = interp->monitoring_matrix.tools[e]; + if (tools == 0) { + continue; + } + int err = PyDict_SetItemString(res, event_names[e], PyLong_FromLong(tools)); + if (err < 0) { + Py_DECREF(res); + return NULL; + } + } + return res; +} + +static PyMethodDef methods[] = { + MONITORING_USE_TOOL_METHODDEF + MONITORING_FREE_TOOL_METHODDEF + MONITORING_GET_TOOL_METHODDEF + MONITORING_REGISTER_CALLBACK_METHODDEF + MONITORING_GET_EVENTS_METHODDEF + MONITORING_SET_EVENTS_METHODDEF + MONITORING_RESTART_EVENTS_METHODDEF + MONITORING__ALL_EVENTS_METHODDEF + {NULL, NULL} // sentinel +}; + +static struct PyModuleDef monitoring_module = { + PyModuleDef_HEAD_INIT, + "sys.monitoring", + NULL, + -1, /* multiple "initialization" just copies the module dict. */ + methods, + NULL, + NULL, + NULL, + NULL +}; + PyObject *_Py_CreateMonitoringObject(void) { PyObject *mod = _PyModule_CreateInitialized(&monitoring_module, PYTHON_API_VERSION); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 0054bb43aa658e..32cb0d6bb5c711 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -36,10 +36,15 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } + if (tstate->tracing) { + Py_RETURN_NONE; + } PyFrameObject* frame = PyEval_GetFrame(); Py_INCREF(frame); assert(frame != NULL); + tstate->tracing++; int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); + tstate->tracing--; Py_DECREF(frame); if (err) { return NULL; @@ -47,7 +52,6 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } - static PyObject * sys_profile_func2( _PyLegacyEventHandler *self, PyObject *const *args, diff --git a/Python/pystate.c b/Python/pystate.c index 91ad6cdbf2cf27..5504fb5a053b05 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -816,6 +816,7 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->monitoring = 0; tstate->_initialized = 1; } From 7971979b4f3c8b2daf4a98292dc46b2b948730dd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Dec 2022 12:34:23 +0000 Subject: [PATCH 006/116] Fix legacy tracing. --- Python/bytecodes.c | 24 +++++++++++------------- Python/generated_cases.c.h | 24 +++++++++++------------- Python/legacy_tracing.c | 5 ++--- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 64b4e88664c320..696c449234b54d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2999,19 +2999,17 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - if (!PyFunction_Check(function) && !PyMethod_Check(function)) { - if (res == NULL) { - _Py_call_instrumentation_exc( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); - } - else { - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); - if (err < 0) { - Py_CLEAR(res); - } + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f8141a93813678..6f72a9cad2c967 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3008,19 +3008,17 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - if (!PyFunction_Check(function) && !PyMethod_Check(function)) { - if (res == NULL) { - _Py_call_instrumentation_exc( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); - } - else { - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); - if (err < 0) { - Py_CLEAR(res); - } + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); } } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 32cb0d6bb5c711..ea53249d33309b 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -83,7 +83,7 @@ sys_profile_call_or_return( Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); PyObject *callable = args[2]; - if (PyFunction_Check(callable) || PyMethod_Check(callable)) { + if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { Py_RETURN_NONE; } return call_profile_func(self, callable); @@ -223,8 +223,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_C_RETURN) | - (1 << PY_MONITORING_EVENT_C_RAISE); + (1 << PY_MONITORING_EVENT_C_RETURN) | (1 << PY_MONITORING_EVENT_C_RAISE); _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); } else if (tstate->interp->sys_profiling_threads == 0) { From 9896902e6f7b41df8dcd23039470b7644e4fe6ab Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Dec 2022 16:58:58 +0000 Subject: [PATCH 007/116] Implement support for multiple tools. --- Include/cpython/code.h | 27 +- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_instruments.h | 4 + Include/internal/pycore_interp.h | 3 +- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 + Lib/opcode.py | 2 + Lib/test/test_monitoring.py | 242 ++++++--- Objects/codeobject.c | 12 +- Python/bytecodes.c | 16 +- Python/ceval.c | 3 + Python/generated_cases.c.h | 16 +- Python/instrumentation.c | 746 +++++++++++++++++++------- Python/opcode_targets.h | 2 +- Python/pystate.c | 1 + 15 files changed, 800 insertions(+), 281 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 49b2a0e7108665..e4d5ff4782cd29 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -3,6 +3,7 @@ #ifndef Py_LIMITED_API #ifndef Py_CODE_H #define Py_CODE_H + #ifdef __cplusplus extern "C" { #endif @@ -45,6 +46,25 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +typedef struct _PyInstrumentationOffsets { + int8_t tools; + int8_t lines; + int8_t instructions; + int8_t size; +} _PyInstrumentationOffsets; + +typedef union _PyInstrumentationLayout { + _PyInstrumentationOffsets offsets; + int32_t bits; +} _PyInstrumentationLayout; + +typedef struct { + uint64_t monitoring_version; /* current instrumentation version */ + _PyInstrumentationLayout layout; + _Py_MonitoringMatrix monitoring_matrix; + uint8_t *monitoring_data; /* array of data for monitoring */ +} _PyCoInstrumentation; + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -103,13 +123,8 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ + _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - uint64_t _co_instrument_version; /* current instrumentation version */ \ - uint8_t _co_monitoring_data_per_instruction; /* Number of bytes per instruction */ \ - uint8_t _co_line_data_offset; /* Offset in data for line data */ \ - uint8_t _co_opcode_data_offset; /* Offset in data for opcode data */ \ - _Py_MonitoringMatrix _co_monitoring_matrix; \ - uint8_t *_co_monitoring_data; /* array of data for monitoring */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 1c87aa31ddb611..5a2b47bba59e16 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); +PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 781aa568d52508..12dee05b465a92 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -66,6 +66,10 @@ extern int _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +extern int +_Py_call_instrumentation_line(PyThreadState *tstate, + PyCodeObject *code, _Py_CODEUNIT *instr); + extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a5c591dde962ef..e903e116e942f7 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -90,7 +90,7 @@ struct _is { PyInterpreterState *next; uint64_t monitoring_version; - uint64_t last_restart_version;; + uint64_t last_restart_version; struct pythreads { uint64_t next_unique_id; @@ -196,6 +196,7 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; + _PyInstrumentationLayout instrumentation_layout; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 86e7d7bb9511b4..ec8ae10c0a3f78 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -138,6 +138,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_STAR] = IMPORT_STAR, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, @@ -488,7 +489,7 @@ static const char *const _PyOpcode_OpName[263] = { [247] = "<247>", [248] = "<248>", [249] = "<249>", - [250] = "<250>", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [251] = "<251>", [252] = "<252>", [253] = "<253>", @@ -576,7 +577,6 @@ static const char *const _PyOpcode_OpName[263] = { case 247: \ case 248: \ case 249: \ - case 250: \ case 251: \ case 252: \ case 253: \ diff --git a/Include/opcode.h b/Include/opcode.h index 70f8b1ea9a3939..619ebfe1cab07b 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -125,6 +125,7 @@ extern "C" { #define INSTRUMENTED_RETURN_VALUE 242 #define INSTRUMENTED_YIELD_VALUE 243 #define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_LINE 250 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index fb240a7472ab86..7a9f3408ff1b69 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -229,6 +229,8 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_YIELD_VALUE', 243) def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) +def_op('INSTRUMENTED_LINE', 250) + hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 54596148b15913..de8e8611b6b2d9 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -23,8 +23,9 @@ def g1(): for _ in gen(): pass -TEST_TOOL = 3 -TEST_TOOL2 = 4 +TEST_TOOL = 2 +TEST_TOOL2 = 3 +TEST_TOOL3 = 4 class MonitoringBaseTest(unittest.TestCase): @@ -178,7 +179,19 @@ def nested_call(): PY_CALLABLES = (types.FunctionType, types.MethodType) -class MonitoringEventsBase: +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + +class MonitoringEventsBase(MonitoringTestBase): def gather_events(self, func): events = [] @@ -201,7 +214,6 @@ def record_call(code, offset, obj): sys.monitoring.set_events(TEST_TOOL, 0) #Remove the final event, the call to `sys.monitoring.set_events` events = events[:-1] - sys.monitoring.set_events(TEST_TOOL, 0) return events def check_events(self, func, expected=None): @@ -275,8 +287,8 @@ def call(code, offset, callable): testfunc() sys.monitoring.set_events(TEST_TOOL, 0) self.assertEqual(errors, []) - self.assertEqual(len(seen), 9) self.assertEqual(stack, [sys._getframe()]) + self.assertEqual(len(seen), 9) class CounterWithDisable: @@ -288,82 +300,162 @@ def __call__(self, *args): if self.disable: return sys.monitoring.DISABLE -class MontoringDisableAndRestartTest(unittest.TestCase): +class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): def test_disable(self): - counter = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - self.assertEqual(counter.count, 0) - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - counter.disable = True - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - counter.count = 0 - f1() - self.assertEqual(counter.count, 0) - sys.monitoring.set_events(TEST_TOOL, 0) + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() def test_restart(self): - counter = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - counter.disable = True - f1() - counter.count = 0 - f1() - self.assertEqual(counter.count, 0) - sys.monitoring.restart_events() - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - - -class MultipleMonitors(unittest.TestCase): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + +class MultipleMonitors(unittest.TestCase, MonitoringTestBase): def test_two_same(self): - counter1 = CounterWithDisable() - counter2 = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - sys.monitoring.set_events(TEST_TOOL2, E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) - counter1.count = 0 - counter2.count = 0 - f1() - count1 = counter1.count - count2 = counter2.count - self.assertEqual(count1, 1) - self.assertEqual(count2, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.set_events(TEST_TOOL2, 0) - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_three_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + counter3 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, counter3) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + sys.monitoring.set_events(TEST_TOOL3, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL3), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2) | (1 << TEST_TOOL3)}) + counter1.count = 0 + counter2.count = 0 + counter3.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + count3 = counter3.count + self.assertEqual((count1, count2, count3), (1, 1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL3, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) def test_two_different(self): - counter1 = CounterWithDisable() - counter2 = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) - counter1.count = 0 - counter2.count = 0 - f1() - count1 = counter1.count - count2 = counter2.count - self.assertEqual(count1, 1) - self.assertEqual(count2, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.set_events(TEST_TOOL2, 0) - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': 1 << TEST_TOOL, 'PY_RETURN': 1 << TEST_TOOL2}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_with_disable(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + counter1.disable = True + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (0, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a6a8dd2e475051..d9a62934edb102 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -339,10 +339,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - co->_co_monitoring_matrix.tools[e] = 0; - } - co->_co_monitoring_data = 0; - co->_co_monitoring_data_per_instruction = 0; + co->_co_instrumentation.monitoring_matrix.tools[e] = 0; + } + co->_co_instrumentation.layout.offsets.size = 0; + co->_co_instrumentation.layout.offsets.tools = -1; + co->_co_instrumentation.layout.offsets.lines = -1; + co->_co_instrumentation.layout.offsets.instructions = -1; + co->_co_instrumentation.monitoring_data = NULL; + co->_co_instrumentation.monitoring_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 696c449234b54d..9cb561f2fbf9b7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -111,7 +111,7 @@ dummy_func( inst(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -131,7 +131,7 @@ dummy_func( assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); next_instr--; @@ -3768,6 +3768,18 @@ dummy_func( PEEK(oparg) = top; } + inst(INSTRUMENTED_LINE) { + int original_opcode = _Py_call_instrumentation_line( + tstate, frame->f_code, next_instr-1); + assert(original_opcode!= 0); + ERROR_IF(original_opcode < 0, error); + opcode = original_opcode; + assert(_PyOpcode_Deopt[opcode] == opcode); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_GOTO(); + } + // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); diff --git a/Python/ceval.c b/Python/ceval.c index dee1763ae6af7d..6aa95f32d870f1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -986,6 +986,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } + /* Because this avoids the RESUME, + * we need to update instrumentation */ + _Py_Instrument(frame->f_code, tstate->interp); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6f72a9cad2c967..ce731770d9b06c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,7 +8,7 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -29,7 +29,7 @@ assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; next_instr--; @@ -3778,6 +3778,18 @@ DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + int original_opcode = _Py_call_instrumentation_line( + tstate, frame->f_code, next_instr-1); + assert(original_opcode!= 0); + if (original_opcode < 0) goto error; + opcode = original_opcode; + assert(_PyOpcode_Deopt[opcode] == opcode); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_GOTO(); + } + TARGET(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c7cf44efd4d38d..c2a474eea7cd22 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -44,6 +44,7 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, /* [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -56,19 +57,228 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { */ }; +_PyInstrumentationOffsets +get_offsets(PyCodeObject *code) { + return code->_co_instrumentation.layout.offsets; +} + +uint8_t * +get_instruction_data(PyCodeObject * code, int offset) +{ + _PyInstrumentationOffsets offsets = get_offsets(code); + return &code->_co_instrumentation.monitoring_data[offset*offsets.size]; +} + +static inline uint8_t +get_tools(PyCodeObject * code, int index, int event) +{ + uint8_t tools; + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + + _PyInstrumentationOffsets offsets = get_offsets(code); + + if (event < PY_MONITORING_EVENT_PY_THROW && offsets.tools >= 0) { + assert(offsets.tools == 0); + tools = get_instruction_data(code, index)[0]; + } + else { + tools = code->_co_instrumentation.monitoring_matrix.tools[event]; + } + assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_matrix.tools[event]) == tools); + return tools; +} + +static inline bool +is_instrumented(int opcode) { + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + static void -de_instrument(_Py_CODEUNIT *instr) +de_instrument(PyCodeObject *code, int offset, int event) { + assert(event != PY_MONITORING_EVENT_LINE); + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(INSTRUMENTED_OPCODES[opcode] == opcode); - int base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - assert(base_opcode != 0); - *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (!is_instrumented(opcode)) { + return; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + int base_opcode; + if (deinstrumented) { + base_opcode = _PyOpcode_Deopt[deinstrumented]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + } + else { + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + assert(opcode == INSTRUMENTED_LINE); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + int orignal_opcode = *orignal_opcode_ptr; + base_opcode = _PyOpcode_Deopt[orignal_opcode]; + assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); + assert(base_opcode != 0); + *orignal_opcode_ptr = base_opcode; + } if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); } } +static void +de_instrument_line(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + assert(opcode == INSTRUMENTED_LINE); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + int orignal_opcode = *orignal_opcode_ptr; + if (is_instrumented(orignal_opcode)) { + /* Instrumented original */ + *instr = _Py_MAKECODEUNIT(orignal_opcode, _Py_OPARG(*instr)); + } + else { + int base_opcode = _PyOpcode_Deopt[orignal_opcode]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[base_opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } +} + +static void +instrument(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_LINE) { + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + opcode = *orignal_opcode_ptr; + assert(!is_instrumented(opcode)); + *orignal_opcode_ptr = INSTRUMENTED_OPCODES[opcode]; + } + else if (!is_instrumented(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + int instrumented = INSTRUMENTED_OPCODES[opcode]; + assert(instrumented); + *instr = _Py_MAKECODEUNIT(instrumented, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } +} + +static void +instrument_line(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + assert(get_offsets(code).lines >= 0); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + if (is_instrumented(opcode)) { + *orignal_opcode_ptr = opcode; + } + else { + assert(_PyOpcode_Deopt[opcode] != 0); + *orignal_opcode_ptr = _PyOpcode_Deopt[opcode]; + } + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_LINE, _Py_OPARG(*instr)); +} + +static void +remove_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyInstrumentationOffsets offsets = get_offsets(code); + if (offsets.tools < 0) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + de_instrument(code, offset, event); + } + } + else { + assert(offsets.tools == 0); + uint8_t *toolsptr = get_instruction_data(code, offset); + *toolsptr &= ~tools; + if (*toolsptr == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + de_instrument(code, offset, event); + } + } +} + +static void +remove_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert (code->_co_instrumentation.layout.offsets.lines >= 0); + if (get_offsets(code).tools < 0) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_line(code, offset); + } + } + else { + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *toolsptr = instr_data + get_offsets(code).lines + 2; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_line(code, offset); + } + } +} + + +static void +add_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyInstrumentationOffsets offsets = get_offsets(code); + if (offsets.tools < 0) { + /* Single tool */ + assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); + assert(_Py_popcount32(tools) == 1); + } + else { + assert(offsets.tools == 0); + uint8_t *toolsptr = get_instruction_data(code, offset); + *toolsptr |= tools; + } + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + instrument(code, offset); + } +} + +static void +add_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert (get_offsets(code).lines >= 0); + if (code->_co_instrumentation.layout.offsets.tools < 0) { + assert(PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == (1 << PY_MONITORING_EVENT_LINE)); + assert(_Py_popcount32(tools) == 1); + /* Single tool */ + instrument_line(code, offset); + } + else { + uint8_t *toolsptr = get_instruction_data(code, offset) + get_offsets(code).lines + 2; + instrument_line(code, offset); + *toolsptr |= tools; + } +} + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( @@ -76,19 +286,16 @@ call_one_instrument( Py_ssize_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); - int tool_bit = 1 << tool; - if (tstate->monitoring & tool_bit) { + if (tstate->tracing) { return 0; } PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; } - uint8_t old_monitoring = tstate->monitoring; - tstate->monitoring |= tool_bit; + tstate->tracing++; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - assert((tstate->monitoring & ~tool_bit) == old_monitoring); - tstate->monitoring = old_monitoring; + tstate->tracing--; if (res == NULL) { return -1; } @@ -96,46 +303,38 @@ call_one_instrument( return (res == &DISABLE); } -static const int8_t LEFT_MOST_BITS[16] = { - -1, 0, 1, 1, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, +static const int8_t MOST_SIGNIFICANT_BITS[16] = { + -1, 0, 1, 1, + 2, 2, 2, 2, + 3, 3, 3, 3, + 3, 3, 3, 3, }; /* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, and can perform relatively poorly on platforms without the necessary intrinsics. */ -static int most_significant_bit(uint8_t bits) { +static inline int most_significant_bit(uint8_t bits) { assert(bits != 0); if (bits > 15) { - return LEFT_MOST_BITS[bits>>4]+4; + return MOST_SIGNIFICANT_BITS[bits>>4]+4; } else { - return LEFT_MOST_BITS[bits]; + return MOST_SIGNIFICANT_BITS[bits]; } } static int -call_multiple_instruments( - PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, +call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); - uint8_t *toolsptr; - uint8_t tools; - if (event < PY_MONITORING_EVENT_PY_THROW) { - toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset]; - tools = *toolsptr; - } - else { - toolsptr = NULL; - tools = code->_co_monitoring_matrix.tools[event]; - } - assert((tools & interp->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_monitoring_matrix.tools[event]) == tools); - /* No line or instruction yet -- Remove this assert when adding line/instruction monitoring */ - assert(code->_co_monitoring_data_per_instruction == 1); + uint8_t tools = get_tools(code, offset, event); + /* No per-instruction monitoring yet */ + assert(code->_co_instrumentation.layout.offsets.instructions < 0); + // assert(tools); while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); @@ -146,41 +345,23 @@ call_multiple_instruments( /* error */ return -1; } - else if (event < PY_MONITORING_EVENT_PY_THROW) { + else { /* DISABLE */ - *toolsptr &= ~(1 << tool); - if (*toolsptr == 0) { - de_instrument(instr); - } + remove_tools(code, offset, event, 1 << tool); } } return 0; } -static int -call_instrument( - PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { - assert(!_PyErr_Occurred(tstate)); - PyInterpreterState *interp = tstate->interp; - if (code->_co_monitoring_data) { - return call_multiple_instruments(interp, tstate, code, event, args, nargsf, instr); - } - uint8_t tools = code->_co_monitoring_matrix.tools[event]; - if (tools == 0) { - return 0; - } - int sole_tool = most_significant_bit(code->_co_monitoring_matrix.tools[event]); - assert(sole_tool >= 0); - assert(_Py_popcount32(code->_co_monitoring_matrix.tools[event]) == 1); - int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); - if (res > 0) { - /* DISABLE */ - de_instrument(instr); - res = 0; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } } - return res; + return true; } int @@ -189,6 +370,8 @@ _Py_call_instrumentation( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; @@ -203,6 +386,8 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -220,6 +405,8 @@ _Py_call_instrumentation_exc( PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -237,6 +424,78 @@ _Py_call_instrumentation_exc( Py_DECREF(instruction_offset_obj); } +/* Line delta. + * 8 bit value. + * if line_delta != -128: + * line = first_line + (offset >> 3) + line_delta; + * else: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + */ + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + int delta = line - code->co_firstlineno - (offset >> 3); + if (delta == (int8_t)delta) { + return delta; + } + return -128; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta == -128) { + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } + return code->co_firstlineno + (offset >> 3) + line_delta; +} + +int +_Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) +{ + PyInterpreterState *interp = tstate->interp; + int offset = instr - _PyCode_CODE(code); + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; + _PyInstrumentationLayout layout = instrumentation->layout; + uint8_t *line_data = &instrumentation->monitoring_data[layout.offsets.size*offset + layout.offsets.lines]; + uint8_t original_opcode = line_data[0]; + int8_t line_delta = ((int8_t *)(char *)line_data)[1]; + int line = compute_line(code, offset, line_delta); + uint8_t tools = layout.offsets.tools >= 0 ? line_data[2] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + PyObject *line_obj = PyLong_FromSsize_t(line); + if (line_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_LINE); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(line_obj); + return -1; + } + else { + /* DISABLE */ + remove_line_tools(code, offset, 1 << tool); + } + } + Py_DECREF(line_obj); + return original_opcode; +} + PyObject * _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { @@ -309,7 +568,7 @@ matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) { assert(0 <= event && event < PY_MONITORING_EVENTS); assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(0 <= val && val <= 1); + assert(val == 0 || val == 1); m->tools[event] &= ~(1 << tool); m->tools[event] |= (val << tool); } @@ -326,73 +585,216 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } +static bool +is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + +static inline _PyInstrumentationLayout +layout_from_flags(bool multitools, bool lines, bool instrs) { + _PyInstrumentationLayout result; + int size = multitools; + result.offsets.tools = multitools - 1; + result.offsets.lines = -1; + result.offsets.instructions = -1; + if (lines) { + result.offsets.lines = size; + size += multitools ? 3 : 2; + } + if (instrs) { + result.offsets.instructions = size; + size += multitools ? 2 : 1; + } + result.offsets.size = size; + return result; +} + +static inline int +tools_data_offset() { + return 0; +} + +static void +initialize_tools(PyCodeObject *code, int size) +{ + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + bool instrumented = is_instrumented(opcode); + assert(opcode != INSTRUMENTED_LINE); + if (instrumented) { + opcode = DE_INSTRUMENT[opcode]; + assert(opcode != 0); + } + opcode = _PyOpcode_Deopt[opcode]; + if (OPCODE_HAS_EVENT[opcode]) { + uint8_t *tools_ptr = &code->_co_instrumentation.monitoring_data[i*size]; + if (instrumented) { + int8_t event; + if (opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg != 0; + } + else { + event = EVENT_FOR_OPCODE[opcode]; + assert(event > 0); + } + assert(event >= 0); + *tools_ptr = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(*tools_ptr != 0); + } + else { + *tools_ptr = 0; + } + } + i += _PyOpcode_Caches[opcode]; + } +} + +static void +copy_data(uint8_t *old_data, int old_size, int old_offset, uint8_t *new_data,int new_size, int new_offset, int code_len) +{ + assert(new_size > 0); + assert(old_offset >= 0); + assert(new_offset >= 0); + for (int i = 0; i < code_len; i++) { + new_data[i*new_size + new_offset] = old_data[i*old_size+old_offset]; + } +} + +static void +initialize_lines(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + int size = code->_co_instrumentation.layout.offsets.size; + int offset = code->_co_instrumentation.layout.offsets.lines; + int prev_line = -1; + for (int i = 0; i < code_len; i++) { + uint8_t *line_data = &code->_co_instrumentation.monitoring_data[i*size+offset]; + /* TO DO -- Use line iterator here */ + int line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + if (line != prev_line && line <= 0) { + uint8_t opcode = _Py_OPCODE(_PyCode_CODE(code)[i]); + assert(opcode != 0); + line_data[0] = opcode; + line_data[1] = compute_line_delta(code, i, line); + prev_line = line; + } + else { + line_data[0] = 0; + line_data[1] = -128; + } + } +} + +static void +initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + int size = code->_co_instrumentation.layout.offsets.size; + int offset = code->_co_instrumentation.layout.offsets.lines; + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data[i*size+offset+2] = + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + } +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { - if (interp->monitoring_version == code->_co_instrument_version) { + + if (is_instrumentation_up_to_date(code, interp)) { + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix)); return 0; } + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; /* First establish how much extra space will need, * and free/allocate it if different to what we had before */ /* For now only allow only one tool per event */ - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == 0); assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); - if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { - /* Don't resize if more space than needed, to avoid churn */ - code->_co_monitoring_data = PyMem_Realloc(code->_co_monitoring_data, code_len * interp->required_monitoring_bytes); - if (code->_co_monitoring_data == NULL) { - code->_co_monitoring_data_per_instruction = 0; - PyErr_NoMemory(); - return -1; + _PyInstrumentationLayout new_layout = interp->instrumentation_layout; + _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; + if (new_layout.bits != old_layout.bits) { + uint8_t *old_data = code->_co_instrumentation.monitoring_data; + if (new_layout.offsets.size) { + code->_co_instrumentation.monitoring_data = PyMem_Malloc(code_len * new_layout.offsets.size); + if (code->_co_instrumentation.monitoring_data == NULL) { + PyErr_NoMemory(); + code->_co_instrumentation.monitoring_data = old_data; + return -1; + } + } + else { + code->_co_instrumentation.monitoring_data = NULL; } - if (code->_co_monitoring_data_per_instruction == 0) { - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - int base_opcode = _PyOpcode_Deopt[opcode]; - if (OPCODE_HAS_EVENT[base_opcode]) { - bool instrumented = INSTRUMENTED_OPCODES[opcode] == opcode; - if (instrumented) { - int8_t event; - if (base_opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg; - } - else { - event = EVENT_FOR_OPCODE[base_opcode]; - } - assert(event >= 0); - code->_co_monitoring_data[i] = code->_co_monitoring_matrix.tools[event]; - } - else { - code->_co_monitoring_data[i] = 0; - } + if (new_layout.offsets.tools >= 0) { + if (restarted) { + uint8_t zero = 0; + copy_data(&zero, 0, 0, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, 0, code_len); + } else if (old_layout.offsets.tools < 0) { + initialize_tools(code, new_layout.offsets.size); + } + else { + copy_data(old_data, old_layout.offsets.size, + 0, code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, 0, code_len); + } + } + if (new_layout.offsets.lines >= 0) { + if (old_layout.offsets.lines < 0) { + initialize_lines(code, interp); + } + else { + copy_data(old_data, old_layout.offsets.size, + old_layout.offsets.lines+1, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, + new_layout.offsets.lines+1, code_len); + } + if (new_layout.offsets.tools >= 0) { + if (old_layout.offsets.tools < 0) { + initialize_line_tools(code, interp); + } + else { + copy_data(old_data, old_layout.offsets.size, + old_layout.offsets.lines+2, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, + new_layout.offsets.lines+2, code_len); } - i += _PyOpcode_Caches[base_opcode]; } - code->_co_monitoring_data_per_instruction = 1; } - for (int i = code_len*code->_co_monitoring_data_per_instruction; i < code_len*interp->required_monitoring_bytes; i++) { - code->_co_monitoring_data[i] = 0; + assert(new_layout.offsets.instructions < 0); + if (old_data) { + PyMem_Free(old_data); } - code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; + code->_co_instrumentation.layout.bits = new_layout.bits; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); /* Avoid instrumenting code that has been disabled. * Only instrument new events */ - _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); - _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); - code->_co_monitoring_matrix = interp->monitoring_matrix; + _Py_MonitoringMatrix new_events; + _Py_MonitoringMatrix removed_events; - if (interp->last_restart_version > code->_co_instrument_version) { + if (restarted) { + removed_events = code->_co_instrumentation.monitoring_matrix; new_events = interp->monitoring_matrix; - removed_events = (_Py_MonitoringMatrix){ 0 }; } - code->_co_instrument_version = interp->monitoring_version; - assert(matrix_empty(matrix_and(new_events, removed_events))); + else { + removed_events = matrix_sub(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix); + new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_matrix); + assert(matrix_empty(matrix_and(new_events, removed_events))); + } + code->_co_instrumentation.monitoring_matrix = interp->monitoring_matrix; + assert(code->_co_instrumentation.layout.bits == interp->instrumentation_layout.bits); + code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { return 0; } @@ -400,7 +802,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); - int oparg = _Py_OPARG(*instr); int base_opcode; if (DE_INSTRUMENT[opcode]) { base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; @@ -418,32 +819,38 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) event = EVENT_FOR_OPCODE[base_opcode]; assert(event > 0); } - uint8_t new_tools = new_events.tools[event]; - if (new_tools) { - assert(INSTRUMENTED_OPCODES[base_opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[base_opcode], oparg); - if (code->_co_monitoring_data) { - code->_co_monitoring_data[i] |= new_tools; - } - } uint8_t removed_tools = removed_events.tools[event]; if (removed_tools) { - if (code->_co_monitoring_data) { - code->_co_monitoring_data[i] &= ~removed_tools; - if (code->_co_monitoring_data[i] == 0) { - de_instrument(instr); - } - } - else if (INSTRUMENTED_OPCODES[opcode] == opcode) { - /* Cannot remove anything but the only tool */ - assert(code->_co_monitoring_matrix.tools[event] == 0); - assert(_Py_popcount32(removed_tools) == 1); - de_instrument(instr); - } + remove_tools(code, i, event, removed_tools); + } + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + add_tools(code, i, event, new_tools); } } i += _PyOpcode_Caches[base_opcode]; } + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + if (new_line_tools) { + /* Insert line instrumentation */ + int offset = new_layout.offsets.lines; + assert(offset >= 0); + for (int i = 0; i < code_len; i++) { + uint8_t *instr_data = get_instruction_data(code, i); + if (instr_data[get_offsets(code).lines]) { + add_line_tools(code, i, new_line_tools); + } + } + } + uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; + if (removed_line_tools) { + for (int i = 0; i < code_len; i++) { + uint8_t *instr_data = get_instruction_data(code, i); + if (instr_data[get_offsets(code).lines]) { + remove_line_tools(code, i, removed_line_tools); + } + } + } return 0; } @@ -454,6 +861,22 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) #define C_CALL_EVENTS \ (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) + +static void +instrument_all_executing_code_objects(PyInterpreterState *interp) { + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + while (ts) { + _PyInterpreterFrame *frame = ts->cframe->current_frame; + while (frame) { + if (frame->owner != FRAME_OWNED_BY_CSTACK) { + _Py_Instrument(frame->f_code, interp); + } + frame = frame->previous; + } + ts = PyThreadState_Next(ts); + } +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { @@ -463,72 +886,24 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (existing_events == events) { return; } - int multitools = 0; + bool multitools = false; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); uint8_t tools = interp->monitoring_matrix.tools[e]; if (_Py_popcount32(tools) > 1) { - multitools++; - } - } - bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; - bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; - if (multitools) { - if (opcodes) { - interp->required_monitoring_bytes = 6; - } - else if (lines) { - interp->required_monitoring_bytes = 4; - } - else { - interp->required_monitoring_bytes = 1; - } - } - else { - if (opcodes) { - interp->required_monitoring_bytes = 3; - } - else if (lines) { - interp->required_monitoring_bytes = 2; - } - else { - interp->required_monitoring_bytes = 0; + multitools = true; } } interp->monitoring_version++; + bool line_monitoring = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); + bool instruction_monitoring = false; + interp->instrumentation_layout = layout_from_flags(multitools, line_monitoring, instruction_monitoring); - /* Instrument all executing code objects */ - - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - - while (ts) { - _PyInterpreterFrame *frame = ts->cframe->current_frame; - while (frame) { - if (frame->owner != FRAME_OWNED_BY_CSTACK) { - _Py_Instrument(frame->f_code, interp); - } - frame = frame->previous; - } - ts = PyThreadState_Next(ts); - } + instrument_all_executing_code_objects(interp); } -/* List of objects in monitoring namespace - class Event(enum.IntFlag) - def use_tool_id(id)->None - def free_tool_id(id)->None - def get_events(tool_id: int)->Event - def set_events(tool_id: int, event_set: Event)->None - def get_local_events(tool_id: int, code: CodeType)->Event - def set_local_events(tool_id: int, code: CodeType, event_set: Event)->None - def register_callback(tool_id: int, event: Event, func: Callable)->Optional[Callable] - def insert_marker(tool_id: int, code: CodeType, offset: Event)->None - def remove_marker(tool_id: int, code: CodeType, offset: Event)->None - def restart_events()->None - DISABLE: object -*/ - /*[clinic input] module monitoring [clinic start generated code]*/ @@ -722,12 +1097,13 @@ monitoring_restart_events_impl(PyObject *module) /*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/ { /* We want to ensure that: - * last restart version < current version * last restart version > instrumented version for all code objects + * last restart version < current version */ PyInterpreterState *interp = _PyInterpreterState_Get(); interp->last_restart_version = interp->monitoring_version + 1; - interp->monitoring_version += 2; + interp->monitoring_version = interp->last_restart_version + 1; + instrument_all_executing_code_objects(interp); Py_RETURN_NONE; } @@ -839,7 +1215,3 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } - - - - diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9ec1dd4f09b7cf..b5a6712b6077c8 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -249,7 +249,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pystate.c b/Python/pystate.c index 5504fb5a053b05..fa6aa31d3c4f85 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -314,6 +314,7 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } + interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ -1, -1, -1, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From f432a66ad6dc1952cfc5241e2a7e8785bfa869ef Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 22 Dec 2022 16:18:26 +0000 Subject: [PATCH 008/116] Get support for line tracing mostly working. Needs to wait for register VM. --- Include/cpython/code.h | 18 +- Include/cpython/pystate.h | 4 +- Include/internal/pycore_code.h | 2 + Include/internal/pycore_instruments.h | 7 + Include/internal/pycore_opcode.h | 32 +- Include/opcode.h | 10 +- Lib/opcode.py | 15 +- Lib/test/test__opcode.py | 6 + Lib/test/test_dis.py | 2 +- Lib/test/test_monitoring.py | 37 +- Lib/test/test_sys_settrace.py | 12 +- Objects/codeobject.c | 16 +- Objects/frameobject.c | 3 + Python/bytecodes.c | 79 +++- Python/ceval.c | 1 + Python/generated_cases.c.h | 84 ++++- Python/instrumentation.c | 499 ++++++++++++++------------ Python/legacy_tracing.c | 162 ++++++++- Python/opcode_targets.h | 18 +- Python/pystate.c | 2 +- 20 files changed, 712 insertions(+), 297 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index e4d5ff4782cd29..17c201f713b56a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -47,10 +47,9 @@ typedef struct { } _PyCoCached; typedef struct _PyInstrumentationOffsets { - int8_t tools; + int8_t multi_tools; int8_t lines; int8_t instructions; - int8_t size; } _PyInstrumentationOffsets; typedef union _PyInstrumentationLayout { @@ -58,11 +57,24 @@ typedef union _PyInstrumentationLayout { int32_t bits; } _PyInstrumentationLayout; +typedef struct { + uint8_t original_opcode; + int8_t line_delta; +} _PyCoLineInstrumentationData; + +typedef struct { + uint8_t *tools; + _PyCoLineInstrumentationData *lines; + uint8_t *line_tools; + uint8_t *per_instruction_opcode; + uint8_t *per_instruction_tools; +} _PyCoInstrumentationData; + typedef struct { uint64_t monitoring_version; /* current instrumentation version */ _PyInstrumentationLayout layout; _Py_MonitoringMatrix monitoring_matrix; - uint8_t *monitoring_data; /* array of data for monitoring */ + _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ } _PyCoInstrumentation; // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3515d7e04b1a64..34518a347de2a4 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -57,8 +57,8 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); typedef struct { - PyCodeObject *code; // The code object for the bounds. May be NULL. - PyCodeAddressRange bounds; // Only valid if code != NULL. + PyCodeObject *code; // The code object for the line number. May be NULL. + int line; // Only valid if code != NULL. } PyTraceInfo; // Internal structure: you should not use it directly, but use public functions diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 81f0dd3dd4811d..d06bd4bddc4d73 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -468,6 +468,8 @@ _Py_MakeShimCode(const _PyShimCodeDef *code); extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); +extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 12dee05b465a92..98cba9884b4f60 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,6 +70,11 @@ extern int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr); +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); @@ -78,6 +83,8 @@ extern void _Py_call_instrumentation_exc(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index); #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index ec8ae10c0a3f78..d8cad9b5f4c2c0 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -138,7 +138,15 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_STAR] = IMPORT_STAR, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, @@ -484,16 +492,16 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [245] = "<245>", - [246] = "<246>", - [247] = "<247>", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = "INSTRUMENTED_JUMP_IF_FALSE_OR_POP", [248] = "<248>", - [249] = "<249>", + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [251] = "<251>", - [252] = "<252>", - [253] = "<253>", - [254] = "<254>", [DO_TRACING] = "DO_TRACING", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", @@ -572,15 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 237: \ case 238: \ case 239: \ - case 245: \ - case 246: \ - case 247: \ case 248: \ - case 249: \ - case 251: \ - case 252: \ - case 253: \ - case 254: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index 619ebfe1cab07b..303872730a43f9 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -125,7 +125,15 @@ extern "C" { #define INSTRUMENTED_RETURN_VALUE 242 #define INSTRUMENTED_YIELD_VALUE 243 #define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_LINE 250 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_JUMP_IF_FALSE_OR_POP 247 +#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 249 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 250 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 251 +#define INSTRUMENTED_POP_JUMP_IF_NONE 252 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index 7a9f3408ff1b69..cfe853250290e3 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -228,9 +228,18 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_RETURN_VALUE', 242) def_op('INSTRUMENTED_YIELD_VALUE', 243) def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) - -def_op('INSTRUMENTED_LINE', 250) - +def_op('INSTRUMENTED_JUMP_FORWARD', 245) +def_op('INSTRUMENTED_JUMP_BACKWARD', 246) +def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) +def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 249) +def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 250) +def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) +def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 252) +def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 253) + +def_op('INSTRUMENTED_LINE', 254) + +# 255 is reserved. hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index db831069c7aeb8..4e5860dfbe6ca3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,6 +20,9 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= 240: + # Opcodes 240 and up are internal instrumentation instructions + continue with self.subTest(opname=name): if code not in has_arg: stack_effect(code) @@ -51,6 +54,9 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= 240: + # Opcodes 240 and up are internal instrumentation instructions + continue with self.subTest(opname=name): if code not in has_arg: common = stack_effect(code) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 950af3ceb24fea..abbf8bae420918 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -893,7 +893,7 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', - ]) + 'INSTRUMENTED_CALL_FUNCTION_EX']) for opcode, opname in enumerate(dis.opname): if opname in long_opcodes: continue diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index de8e8611b6b2d9..bab69d55446dcc 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -292,14 +292,30 @@ def call(code, offset, callable): class CounterWithDisable: + def __init__(self): self.disable = False self.count = 0 + def __call__(self, *args): self.count += 1 if self.disable: return sys.monitoring.DISABLE + +class RecorderWithDisable: + + def __init__(self, events): + self.disable = False + self.events = events + + def __call__(self, code, event): + self.events.append(event) + print(len(self.events)) + if self.disable: + return sys.monitoring.DISABLE + + class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): def test_disable(self): @@ -341,7 +357,7 @@ def test_restart(self): sys.monitoring.restart_events() -class MultipleMonitors(unittest.TestCase, MonitoringTestBase): +class MultipleMonitorsTest(unittest.TestCase, MonitoringTestBase): def test_two_same(self): try: @@ -459,3 +475,22 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() +class LineMontoringTest(unittest.TestCase): + + def test_lines_single(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + counter = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, counter) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(events, [487, 12, 488]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..35b1a0b629d406 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1974,12 +1974,12 @@ def test_no_jump_over_return_try_finally_in_finally_block(output): pass output.append(12) - @jump_test(3, 4, [1], (ValueError, 'after')) - def test_no_jump_infinite_while_loop(output): - output.append(1) - while True: - output.append(3) - output.append(4) + #@jump_test(3, 4, [1], (ValueError, 'after')) + #def test_no_jump_infinite_while_loop(output): + #output.append(1) + #while True: + #output.append(3) + #output.append(4) @jump_test(2, 4, [4, 4]) def test_jump_forwards_into_while_block(output): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d9a62934edb102..733eb1a6a1546d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -341,10 +341,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) for (int e = 0; e < PY_MONITORING_EVENTS; e++) { co->_co_instrumentation.monitoring_matrix.tools[e] = 0; } - co->_co_instrumentation.layout.offsets.size = 0; - co->_co_instrumentation.layout.offsets.tools = -1; - co->_co_instrumentation.layout.offsets.lines = -1; - co->_co_instrumentation.layout.offsets.instructions = -1; + co->_co_instrumentation.layout.offsets.multi_tools = 0; + co->_co_instrumentation.layout.offsets.lines = 0; + co->_co_instrumentation.layout.offsets.instructions = 0; co->_co_instrumentation.monitoring_data = NULL; co->_co_instrumentation.monitoring_version = 0; /* not set */ @@ -1459,11 +1458,12 @@ PyCode_GetFreevars(PyCodeObject *code) } static void -deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) +deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { + Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { _Py_CODEUNIT instruction = instructions[i]; - int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); while (caches--) { @@ -1486,7 +1486,7 @@ _PyCode_GetCode(PyCodeObject *co) if (code == NULL) { return NULL; } - deopt_code((_Py_CODEUNIT *)PyBytes_AS_STRING(code), Py_SIZE(co)); + deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); assert(co->_co_cached->_co_code == NULL); co->_co_cached->_co_code = Py_NewRef(code); return code; @@ -2216,7 +2216,7 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Fini(PyCodeObject *co) { - deopt_code(_PyCode_CODE(co), Py_SIZE(co)); + deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); if (co->_co_cached != NULL) { Py_CLEAR(co->_co_cached->_co_code); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 4861f80a0ca88e..84a205ecaddf9d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -819,6 +819,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { + fprintf(stderr, "Popping exception\n"); /* Pop exception stack as well as the evaluation stack */ PyThreadState *tstate = _PyThreadState_GET(); _PyErr_StackItem *exc_info = tstate->exc_info; @@ -829,11 +830,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore Py_XDECREF(value); } else { + fprintf(stderr, "Popping object\n"); PyObject *v = _PyFrame_StackPop(f->f_frame); Py_XDECREF(v); } start_stack = pop_value(start_stack); } + fprintf(stderr, "Jumping to %d\n", best_addr); /* Finally set the new lasti and return OK. */ f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9cb561f2fbf9b7..d1381f37292149 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3769,17 +3769,88 @@ dummy_func( } inst(INSTRUMENTED_LINE) { + _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame->f_code, next_instr-1); - assert(original_opcode!= 0); ERROR_IF(original_opcode < 0, error); + next_instr--; + if (frame->prev_instr != next_instr) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; - assert(_PyOpcode_Deopt[opcode] == opcode); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); DISPATCH_GOTO(); } + // stack effect: ( -- ) + inst(INSTRUMENTED_JUMP_FORWARD) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + JUMPBY(oparg); + } + + // stack effect: ( -- ) + inst(INSTRUMENTED_JUMP_BACKWARD) { + assert(oparg < INSTR_OFFSET()); + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + } + + inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + } + + inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + } + + inst(INSTRUMENTED_POP_JUMP_IF_TRUE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_FALSE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + } + // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); diff --git a/Python/ceval.c b/Python/ceval.c index 6aa95f32d870f1..7025e9bd641a4a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1115,6 +1115,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ + assert(0); opcode = _Py_OPCODE(*next_instr); _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ce731770d9b06c..4238b2776403ac 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2390,6 +2390,7 @@ } TARGET(POP_JUMP_IF_TRUE) { + PREDICTED(POP_JUMP_IF_TRUE); PyObject *cond = POP(); if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); @@ -2413,6 +2414,7 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { + PREDICTED(POP_JUMP_IF_NOT_NONE); PyObject *value = POP(); if (!Py_IsNone(value)) { JUMPBY(oparg); @@ -2422,6 +2424,7 @@ } TARGET(POP_JUMP_IF_NONE) { + PREDICTED(POP_JUMP_IF_NONE); PyObject *value = POP(); if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); @@ -2434,6 +2437,7 @@ } TARGET(JUMP_IF_FALSE_OR_POP) { + PREDICTED(JUMP_IF_FALSE_OR_POP); PyObject *cond = TOP(); int err; if (Py_IsTrue(cond)) { @@ -2460,6 +2464,7 @@ } TARGET(JUMP_IF_TRUE_OR_POP) { + PREDICTED(JUMP_IF_TRUE_OR_POP); PyObject *cond = TOP(); int err; if (Py_IsFalse(cond)) { @@ -3779,17 +3784,88 @@ } TARGET(INSTRUMENTED_LINE) { + _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame->f_code, next_instr-1); - assert(original_opcode!= 0); if (original_opcode < 0) goto error; + next_instr--; + if (frame->prev_instr != next_instr) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; - assert(_PyOpcode_Deopt[opcode] == opcode); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); DISPATCH_GOTO(); } + TARGET(INSTRUMENTED_JUMP_FORWARD) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(oparg < INSTR_OFFSET()); + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_IF_TRUE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + } + + TARGET(INSTRUMENTED_JUMP_IF_FALSE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + } + TARGET(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c2a474eea7cd22..ba53083a55edbc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -20,7 +20,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_CALL] = CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, -/* [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -29,10 +28,8 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, -*/ }; - static const uint8_t INSTRUMENTED_OPCODES[256] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [CALL] = INSTRUMENTED_CALL, @@ -45,7 +42,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, - /* [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -54,7 +50,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - */ }; _PyInstrumentationOffsets @@ -62,25 +57,15 @@ get_offsets(PyCodeObject *code) { return code->_co_instrumentation.layout.offsets; } -uint8_t * -get_instruction_data(PyCodeObject * code, int offset) -{ - _PyInstrumentationOffsets offsets = get_offsets(code); - return &code->_co_instrumentation.monitoring_data[offset*offsets.size]; -} - static inline uint8_t get_tools(PyCodeObject * code, int index, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - - _PyInstrumentationOffsets offsets = get_offsets(code); - - if (event < PY_MONITORING_EVENT_PY_THROW && offsets.tools >= 0) { - assert(offsets.tools == 0); - tools = get_instruction_data(code, index)[0]; + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + tools = monitoring->tools[index]; } else { tools = code->_co_instrumentation.monitoring_matrix.tools[event]; @@ -95,6 +80,25 @@ is_instrumented(int opcode) { return INSTRUMENTED_OPCODES[opcode] == opcode; } +int _Py_GetBaseOpcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + /* To do -- Handle INSTRUMENTED_INSTRUCTION + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcode[offset]; + } + */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + assert(_PyOpcode_Deopt[opcode]); + return _PyOpcode_Deopt[opcode]; +} + static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -114,13 +118,12 @@ de_instrument(PyCodeObject *code, int offset, int event) else { /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ assert(opcode == INSTRUMENTED_LINE); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - int orignal_opcode = *orignal_opcode_ptr; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int orignal_opcode = lines->original_opcode; base_opcode = _PyOpcode_Deopt[orignal_opcode]; assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); assert(base_opcode != 0); - *orignal_opcode_ptr = base_opcode; + lines->original_opcode = base_opcode; } if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); @@ -133,22 +136,26 @@ de_instrument_line(PyCodeObject *code, int offset) /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(opcode == INSTRUMENTED_LINE); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - int orignal_opcode = *orignal_opcode_ptr; - if (is_instrumented(orignal_opcode)) { + if (opcode != INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int original_opcode = lines->original_opcode; + assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + if (is_instrumented(original_opcode)) { /* Instrumented original */ - *instr = _Py_MAKECODEUNIT(orignal_opcode, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(original_opcode, _Py_OPARG(*instr)); } else { - int base_opcode = _PyOpcode_Deopt[orignal_opcode]; + int base_opcode = _PyOpcode_Deopt[original_opcode]; assert(base_opcode != 0); *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); } } + /* Mark instruction as candidate for line instrumentation */ + lines->original_opcode = 255; } static void @@ -158,11 +165,10 @@ instrument(PyCodeObject *code, int offset) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_LINE) { - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - opcode = *orignal_opcode_ptr; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + opcode = lines->original_opcode; assert(!is_instrumented(opcode)); - *orignal_opcode_ptr = INSTRUMENTED_OPCODES[opcode]; + lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; } else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; @@ -178,20 +184,27 @@ instrument(PyCodeObject *code, int offset) static void instrument_line(PyCodeObject *code, int offset) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(get_offsets(code).lines >= 0); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + if (opcode == INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + assert(lines->original_opcode == 255); if (is_instrumented(opcode)) { - *orignal_opcode_ptr = opcode; + lines->original_opcode = opcode; } else { + assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); - *orignal_opcode_ptr = _PyOpcode_Deopt[opcode]; + assert(_PyOpcode_Deopt[opcode] != RESUME); + lines->original_opcode = _PyOpcode_Deopt[opcode]; + } + assert(lines->original_opcode > 0); *instr = _Py_MAKECODEUNIT(INSTRUMENTED_LINE, _Py_OPARG(*instr)); + assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); } static void @@ -199,20 +212,18 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyInstrumentationOffsets offsets = get_offsets(code); - if (offsets.tools < 0) { - /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; - assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + monitoring->tools[offset] &= ~tools; + if (monitoring->tools[offset] == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); } } else { - assert(offsets.tools == 0); - uint8_t *toolsptr = get_instruction_data(code, offset); - *toolsptr &= ~tools; - if (*toolsptr == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); } } @@ -222,19 +233,20 @@ static void remove_line_tools(PyCodeObject * code, int offset, int tools) { assert (code->_co_instrumentation.layout.offsets.lines >= 0); - if (get_offsets(code).tools < 0) { - /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; - assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->line_tools) + { + uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { de_instrument_line(code, offset); } } else { - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *toolsptr = instr_data + get_offsets(code).lines + 2; - *toolsptr &= ~tools; - if (*toolsptr == 0 ) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); } } @@ -246,17 +258,16 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyInstrumentationOffsets offsets = get_offsets(code); - if (offsets.tools < 0) { + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->tools + ) { + code->_co_instrumentation.monitoring_data->tools[offset] |= tools; + } + else { /* Single tool */ assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); assert(_Py_popcount32(tools) == 1); } - else { - assert(offsets.tools == 0); - uint8_t *toolsptr = get_instruction_data(code, offset); - *toolsptr |= tools; - } if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { instrument(code, offset); } @@ -266,17 +277,17 @@ static void add_line_tools(PyCodeObject * code, int offset, int tools) { assert (get_offsets(code).lines >= 0); - if (code->_co_instrumentation.layout.offsets.tools < 0) { - assert(PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == (1 << PY_MONITORING_EVENT_LINE)); - assert(_Py_popcount32(tools) == 1); - /* Single tool */ - instrument_line(code, offset); + assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->line_tools + ) { + code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; } else { - uint8_t *toolsptr = get_instruction_data(code, offset) + get_offsets(code).lines + 2; - instrument_line(code, offset); - *toolsptr |= tools; + /* Single tool */ + assert(_Py_popcount32(tools) == 1); } + instrument_line(code, offset); } /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ @@ -286,9 +297,7 @@ call_one_instrument( Py_ssize_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); - if (tstate->tracing) { - return 0; - } + assert(tstate->tracing == 0); PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; @@ -326,11 +335,14 @@ static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + if (tstate->tracing) { + return 0; + } PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); /* No per-instruction monitoring yet */ - assert(code->_co_instrumentation.layout.offsets.instructions < 0); + assert(!code->_co_instrumentation.layout.offsets.instructions); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -364,13 +376,19 @@ matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) return true; } +static bool +is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + int _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); @@ -386,7 +404,7 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); @@ -396,6 +414,30 @@ _Py_call_instrumentation_arg( return err; } +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target +) { + PyCodeObject *code = frame->f_code; + int from = instr - _PyCode_CODE(code); + int to = target - _PyCode_CODE(code); + PyObject *to_obj = PyLong_FromLong(to); + if (to_obj == NULL) { + return -1; + } + PyObject *from_obj = PyLong_FromLong(from); + if (from_obj == NULL) { + Py_DECREF(to_obj); + return -1; + } + PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(to_obj); + Py_DECREF(from_obj); + return err; +} + void _Py_call_instrumentation_exc( PyThreadState *tstate, int event, @@ -436,36 +478,53 @@ static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { int delta = line - code->co_firstlineno - (offset >> 3); - if (delta == (int8_t)delta) { + if (delta < 128 && delta > -128) { return delta; } - return -128; + return -127; } static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - if (line_delta == -128) { + assert(line_delta != -128); + if (line_delta == -127) { /* Compute from table */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } return code->co_firstlineno + (offset >> 3) + line_delta; } +int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index) +{ + _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; + assert(instrumentation->monitoring_data != NULL); + assert(instrumentation->monitoring_data->lines != NULL); + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; + int8_t line_delta = line_data->line_delta; + int line = compute_line(code, index, line_delta); + return line; +} + int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) { - PyInterpreterState *interp = tstate->interp; - int offset = instr - _PyCode_CODE(code); - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyInstrumentationLayout layout = instrumentation->layout; - uint8_t *line_data = &instrumentation->monitoring_data[layout.offsets.size*offset + layout.offsets.lines]; - uint8_t original_opcode = line_data[0]; - int8_t line_delta = ((int8_t *)(char *)line_data)[1]; + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; + uint8_t original_opcode = line_data->original_opcode; + if (tstate->tracing) { + return original_opcode; + } + PyInterpreterState *interp = tstate->interp; + int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); - uint8_t tools = layout.offsets.tools >= 0 ? line_data[2] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? + code->_co_instrumentation.monitoring_data->line_tools[offset] : + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -476,7 +535,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD assert(tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, + int res = call_one_instrument(interp, tstate, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, tool, PY_MONITORING_EVENT_LINE); if (res == 0) { @@ -493,6 +552,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD } } Py_DECREF(line_obj); + assert(original_opcode != 0); + assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } @@ -517,6 +578,14 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [RESUME] = -1, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -530,6 +599,14 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_RESUME] = true, [YIELD_VALUE] = true, [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, }; static inline _Py_MonitoringMatrix @@ -585,28 +662,12 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -static bool -is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) -{ - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; -} - static inline _PyInstrumentationLayout layout_from_flags(bool multitools, bool lines, bool instrs) { _PyInstrumentationLayout result; - int size = multitools; - result.offsets.tools = multitools - 1; - result.offsets.lines = -1; - result.offsets.instructions = -1; - if (lines) { - result.offsets.lines = size; - size += multitools ? 3 : 2; - } - if (instrs) { - result.offsets.instructions = size; - size += multitools ? 2 : 1; - } - result.offsets.size = size; + result.offsets.multi_tools = multitools; + result.offsets.lines = lines; + result.offsets.instructions = instrs; return result; } @@ -616,21 +677,24 @@ tools_data_offset() { } static void -initialize_tools(PyCodeObject *code, int size) +initialize_tools(PyCodeObject *code) { + uint8_t* tools = code->_co_instrumentation.monitoring_data->tools; + assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + } bool instrumented = is_instrumented(opcode); - assert(opcode != INSTRUMENTED_LINE); if (instrumented) { opcode = DE_INSTRUMENT[opcode]; assert(opcode != 0); } opcode = _PyOpcode_Deopt[opcode]; if (OPCODE_HAS_EVENT[opcode]) { - uint8_t *tools_ptr = &code->_co_instrumentation.monitoring_data[i*size]; if (instrumented) { int8_t event; if (opcode == RESUME) { @@ -642,65 +706,121 @@ initialize_tools(PyCodeObject *code, int size) assert(event > 0); } assert(event >= 0); - *tools_ptr = code->_co_instrumentation.monitoring_matrix.tools[event]; - assert(*tools_ptr != 0); + tools[i] = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(tools[i] != 0); } else { - *tools_ptr = 0; + tools[i] = 0; } } i += _PyOpcode_Caches[opcode]; } } -static void -copy_data(uint8_t *old_data, int old_size, int old_offset, uint8_t *new_data,int new_size, int new_offset, int code_len) +static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { - assert(new_size > 0); - assert(old_offset >= 0); - assert(new_offset >= 0); - for (int i = 0; i < code_len; i++) { - new_data[i*new_size + new_offset] = old_data[i*old_size+old_offset]; + /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ + if (opcode == END_FOR) { + return false; + } + while (bounds->ar_end <= index*(int)sizeof(_Py_CODEUNIT)) { + if (!_PyLineTable_NextAddressRange(bounds)) { + return false; + } } + assert(index*(int)sizeof(_Py_CODEUNIT) >= bounds->ar_start && + index*(int)sizeof(_Py_CODEUNIT) < bounds->ar_end); + return bounds->ar_line >= 0 && bounds->ar_line != current_line; } static void -initialize_lines(PyCodeObject *code, PyInterpreterState *interp) +initialize_lines(PyCodeObject *code) { + _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + assert(line_data != NULL); int code_len = (int)Py_SIZE(code); - int size = code->_co_instrumentation.layout.offsets.size; - int offset = code->_co_instrumentation.layout.offsets.lines; - int prev_line = -1; - for (int i = 0; i < code_len; i++) { - uint8_t *line_data = &code->_co_instrumentation.monitoring_data[i*size+offset]; - /* TO DO -- Use line iterator here */ - int line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - if (line != prev_line && line <= 0) { - uint8_t opcode = _Py_OPCODE(_PyCode_CODE(code)[i]); - assert(opcode != 0); - line_data[0] = opcode; - line_data[1] = compute_line_delta(code, i, line); - prev_line = line; + int current_line = -1; + PyCodeAddressRange range; + _PyCode_InitAddressRange(code, &range); + for (int i = 0; i <= code->_co_firsttraceable; i++) { + line_data[i].original_opcode = 0; + line_data[i].line_delta = -128; + } + for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = 255; + current_line = range.ar_line; + line_data[i].line_delta = compute_line_delta(code, i, current_line); } else { - line_data[0] = 0; - line_data[1] = -128; + /* Mark as not being an instrumentation point */ + line_data[i].original_opcode = 0; + line_data[i].line_delta = compute_line_delta(code, i, current_line); } + i += _PyOpcode_Caches[opcode]; } } static void initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) { + uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; + assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); - int size = code->_co_instrumentation.layout.offsets.size; - int offset = code->_co_instrumentation.layout.offsets.lines; for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data[i*size+offset+2] = - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; } } +int +update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +{ + _PyInstrumentationLayout new_layout = interp->instrumentation_layout; + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; + int code_len = (int)Py_SIZE(code); + if (code->_co_instrumentation.monitoring_data == NULL) { + code->_co_instrumentation.monitoring_data = PyMem_Calloc(sizeof(_PyCoInstrumentationData), 1); + if (code->_co_instrumentation.monitoring_data == NULL) { + PyErr_NoMemory(); + return -1; + } + } + if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->tools == NULL) { + code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->tools == NULL) { + PyErr_NoMemory(); + return -1; + } + if (restarted) { + memset(code->_co_instrumentation.monitoring_data->tools, 0, code_len); + } else { + initialize_tools(code); + } + } + if (new_layout.offsets.lines) { + if (code->_co_instrumentation.monitoring_data->lines == NULL) { + code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_instrumentation.monitoring_data->lines == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_lines(code); + } + if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { + code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_line_tools(code, interp); + } + } + assert(new_layout.offsets.instructions == 0); + code->_co_instrumentation.layout.bits = new_layout.bits; + return 0; +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -710,76 +830,15 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; - /* First establish how much extra space will need, - * and free/allocate it if different to what we had before */ - /* For now only allow only one tool per event */ assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); _PyInstrumentationLayout new_layout = interp->instrumentation_layout; _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; if (new_layout.bits != old_layout.bits) { - uint8_t *old_data = code->_co_instrumentation.monitoring_data; - if (new_layout.offsets.size) { - code->_co_instrumentation.monitoring_data = PyMem_Malloc(code_len * new_layout.offsets.size); - if (code->_co_instrumentation.monitoring_data == NULL) { - PyErr_NoMemory(); - code->_co_instrumentation.monitoring_data = old_data; - return -1; - } - } - else { - code->_co_instrumentation.monitoring_data = NULL; - } - if (new_layout.offsets.tools >= 0) { - if (restarted) { - uint8_t zero = 0; - copy_data(&zero, 0, 0, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, 0, code_len); - } else if (old_layout.offsets.tools < 0) { - initialize_tools(code, new_layout.offsets.size); - } - else { - copy_data(old_data, old_layout.offsets.size, - 0, code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, 0, code_len); - } - } - if (new_layout.offsets.lines >= 0) { - if (old_layout.offsets.lines < 0) { - initialize_lines(code, interp); - } - else { - copy_data(old_data, old_layout.offsets.size, - old_layout.offsets.lines+1, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, - new_layout.offsets.lines+1, code_len); - } - if (new_layout.offsets.tools >= 0) { - if (old_layout.offsets.tools < 0) { - initialize_line_tools(code, interp); - } - else { - copy_data(old_data, old_layout.offsets.size, - old_layout.offsets.lines+2, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, - new_layout.offsets.lines+2, code_len); - } - } - } - assert(new_layout.offsets.instructions < 0); - if (old_data) { - PyMem_Free(old_data); + if (update_instrumentation_data(code, interp)) { + return -1; } - code->_co_instrumentation.layout.bits = new_layout.bits; } - //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", - // code, code->_co_instrument_version, interp->monitoring_version); - /* Avoid instrumenting code that has been disabled. - * Only instrument new events - */ _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; @@ -831,24 +890,20 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyOpcode_Caches[base_opcode]; } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools) { - /* Insert line instrumentation */ - int offset = new_layout.offsets.lines; - assert(offset >= 0); - for (int i = 0; i < code_len; i++) { - uint8_t *instr_data = get_instruction_data(code, i); - if (instr_data[get_offsets(code).lines]) { - add_line_tools(code, i, new_line_tools); - } - } - } uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (removed_line_tools) { - for (int i = 0; i < code_len; i++) { - uint8_t *instr_data = get_instruction_data(code, i); - if (instr_data[get_offsets(code).lines]) { - remove_line_tools(code, i, removed_line_tools); + if (new_line_tools || removed_line_tools) { + _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + if (line_data[i].original_opcode) { + if (removed_line_tools) { + remove_line_tools(code, i, removed_line_tools); + } + if (new_line_tools) { + add_line_tools(code, i, new_line_tools); + } } + int opcode = _Py_GetBaseOpcode(code, i); + i += _PyOpcode_Caches[opcode]; } } return 0; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index ea53249d33309b..bb5b78a177943a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "pycore_ceval.h" #include "pycore_instruments.h" #include "pycore_pyerrors.h" #include "pycore_pymem.h" @@ -14,6 +15,7 @@ typedef struct _PyLegacyEventHandler { PyObject_HEAD vectorcallfunc vectorcall; int event; + PyCodeObject *last_code; } _PyLegacyEventHandler; static void @@ -36,15 +38,10 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } - if (tstate->tracing) { - Py_RETURN_NONE; - } PyFrameObject* frame = PyEval_GetFrame(); Py_INCREF(frame); assert(frame != NULL); - tstate->tracing++; int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); - tstate->tracing--; Py_DECREF(frame); if (err) { return NULL; @@ -90,7 +87,25 @@ sys_profile_call_or_return( } static PyObject * -sys_profile_exception_func( +call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject* frame = PyEval_GetFrame(); + Py_INCREF(frame); + assert(frame != NULL); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sys_trace_exception_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -102,11 +117,33 @@ sys_profile_exception_func( if (arg == NULL) { return NULL; } - PyObject *res = call_profile_func(self, arg); + PyObject *res = call_trace_func(self, arg); Py_DECREF(arg); return res; } +static PyObject * +sys_trace_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + return call_trace_func(self, Py_None); +} + +static PyObject * +sys_trace_func3( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_trace_func(self, args[2]); +} + static PyObject * sys_trace_none_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -115,7 +152,93 @@ sys_trace_none_func( assert(kwnames == NULL); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); - return call_profile_func(self, Py_None); + return call_trace_func(self, Py_None); +} + +static PyObject * +trace_line(PyThreadState *tstate, PyCodeObject *code, + _PyLegacyEventHandler *self, PyFrameObject* frame, int line +) { + Py_INCREF(frame); + tstate->trace_info.code = code; + tstate->trace_info.line = frame->f_lineno = line; + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +sys_trace_line_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + _PyInterpreterFrame *iframe = _PyEval_GetFrame(); + assert(iframe); + PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + assert(args[0] == (PyObject *)iframe->f_code); + int line = _PyLong_AsInt(args[1]); + assert(line >= 0); + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, iframe->f_code, self, frame, line); +} + + +static PyObject * +sys_trace_branch_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + _PyInterpreterFrame *iframe = _PyEval_GetFrame(); + assert(iframe); + PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + int from = _PyLong_AsInt(args[1]); + assert(from >= 0); + int to = _PyLong_AsInt(args[2]); + assert(to >= 0); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + if (from > to) { + /* Backwards jump */ + return trace_line(tstate, iframe->f_code, self, frame, to_line); + } + else { + /* Forwards jump, only handle if new line */ + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (from_line != to_line) { + return trace_line(tstate, iframe->f_code, self, frame, to_line); + } + } + Py_RETURN_NONE; } @@ -262,25 +385,30 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_exception_func, PyTrace_EXCEPTION, + (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, PY_MONITORING_EVENT_RAISE, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_none_func, PyTrace_LINE, + (vectorcallfunc)sys_trace_line_func, PyTrace_LINE, PY_MONITORING_EVENT_LINE, -1)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { @@ -291,14 +419,16 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE); - if (tstate->interp->f_opcode_trace_set) { + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH); + /* TO DO -- opcode events */ + if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); } Py_XDECREF(old_traceobj); return 0; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index b5a6712b6077c8..18c75a50914efc 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -244,15 +244,15 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_RETURN_VALUE, &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + &&_unknown_opcode, + &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/pystate.c b/Python/pystate.c index fa6aa31d3c4f85..460b34a4677f03 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -314,7 +314,7 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } - interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ -1, -1, -1, 0 }; + interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ 0, 0, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 15a1ccd69bf2030bf96f5dfc80edbacccd857b2d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 13 Jan 2023 10:58:17 +0000 Subject: [PATCH 009/116] Fix up INSTRUMENTED_OPCODES vector. --- Python/instrumentation.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 134ab488ae08cf..0a1c2594501d89 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -32,24 +32,33 @@ static const uint8_t DE_INSTRUMENT[256] = { static const uint8_t INSTRUMENTED_OPCODES[256] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [CALL] = INSTRUMENTED_CALL, - [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, }; _PyInstrumentationOffsets @@ -173,6 +182,9 @@ instrument(PyCodeObject *code, int offset) else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; + if (!instrumented) { + printf("OPCODE: %s\n", _PyOpcode_OpName[opcode]); + } assert(instrumented); instr->opcode = instrumented; if (_PyOpcode_Caches[opcode]) { From 9abb339defb6e030f2e35371c137698dd2c4dc50 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 16 Jan 2023 10:48:14 +0000 Subject: [PATCH 010/116] Fix instrumented branches to call instrumentation with correct target. --- Python/bytecodes.c | 86 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 52e3b6cee65ed6..a8c02896568961 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3418,45 +3418,99 @@ dummy_func( } inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = TOP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + err*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + if (err == 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + next_instr = there; } inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = TOP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + (1-err)*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + if (err) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + err*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + (1-err)*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *there; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + there = next_instr + oparg; + } + else { + Py_DECREF(value); + there = next_instr; + } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *there; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + there = next_instr; + } + else { + Py_DECREF(value); + there = next_instr + oparg; + } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + next_instr = there; } // stack effect: ( -- ) From aa09895e86b1f9568616fcaff41375be04ce4978 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 17 Jan 2023 11:55:40 +0000 Subject: [PATCH 011/116] Add PY_THROW event handling and fix up line table. --- Python/ceval.c | 17 ++++++++++++++++- Python/instrumentation.c | 40 ++++++++++++++++++++++++++++++---------- Python/legacy_tracing.c | 19 ++++++++++++++++++- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index d4dabe04920497..1e2a08555e3f3a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -187,6 +187,9 @@ static void monitor_unwind(PyThreadState *tstate, static void monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); @@ -998,6 +1001,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, * we need to update instrumentation */ _Py_Instrument(frame->f_code, tstate->interp); + monitor_throw(tstate, frame, frame->prev_instr); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -1225,9 +1229,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyException_SetTraceback(val, Py_None); Py_XDECREF(tb); Py_XDECREF(exc); - monitor_handled(tstate, frame, next_instr-1, val); PUSH(val); JUMPTO(handler); + monitor_handled(tstate, frame, next_instr, val); /* Resume normal execution */ DISPATCH(); } @@ -2271,6 +2275,17 @@ monitor_handled(PyThreadState *tstate, _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); } +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { + return; + } + _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr, NULL); +} + void PyThreadState_EnterTracing(PyThreadState *tstate) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8424b01d2e9129..6668dc535551aa 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -574,15 +574,23 @@ _Py_call_instrumentation_exc( /* Line delta. * 8 bit value. - * if line_delta != -128: - * line = first_line + (offset >> 3) + line_delta; - * else: + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> 3) + line_delta; */ static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { + if (line < 0) { + return -128; + } + assert(line >= code->co_firstlineno); + assert(offset >= code->_co_firsttraceable); + assert(offset < Py_SIZE(code)); int delta = line - code->co_firstlineno - (offset >> 3); if (delta < 128 && delta > -128) { return delta; @@ -593,12 +601,18 @@ compute_line_delta(PyCodeObject *code, int offset, int line) static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - assert(line_delta != -128); - if (line_delta == -127) { + if (line_delta > -127) { + assert((offset >> 3) + line_delta >= 0); + return code->co_firstlineno + (offset >> 3) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); /* Compute from table */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } - return code->co_firstlineno + (offset >> 3) + line_delta; } int @@ -607,6 +621,8 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; assert(instrumentation->monitoring_data != NULL); assert(instrumentation->monitoring_data->lines != NULL); + assert(index >= code->_co_firsttraceable); + assert(index < Py_SIZE(code)); _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; int8_t line_delta = line_data->line_delta; int line = compute_line(code, index, line_delta); @@ -859,14 +875,18 @@ initialize_lines(PyCodeObject *code) if (is_new_line(i, current_line, &range, opcode)) { line_data[i].original_opcode = 255; current_line = range.ar_line; - line_data[i].line_delta = compute_line_delta(code, i, current_line); } else { /* Mark as not being an instrumentation point */ line_data[i].original_opcode = 0; - line_data[i].line_delta = compute_line_delta(code, i, current_line); } - i += _PyOpcode_Caches[opcode]; + assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); + for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = -128; + } } } @@ -1002,7 +1022,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) int8_t event; if (base_opcode == RESUME) { int oparg = _Py_OPARG(*instr); - event = oparg; + event = oparg > 0; } else { event = EVENT_FOR_OPCODE[base_opcode]; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 4c76c47259cc4d..016c1d99a35420 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -159,6 +159,9 @@ static PyObject * trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, _PyLegacyEventHandler *self, PyFrameObject* frame, int line ) { + if (line < 0) { + Py_RETURN_NONE; + } if (tstate->trace_info.code == iframe->f_code && tstate->trace_info.line == line) { /* Already traced this line */ @@ -229,6 +232,9 @@ sys_trace_handled_exception( int offset = _PyLong_AsInt(args[1]); assert(offset >= 0); int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); + if (line < 0) { + Py_RETURN_NONE; + } return trace_line(tstate, iframe, self, frame, line); } @@ -411,6 +417,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { @@ -426,6 +437,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_LINE, -1)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { @@ -449,7 +465,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); + (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW)); /* TO DO -- opcode events */ if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); From 9e5d87de66c3a1ecc527e6825f86375e006651f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 17 Jan 2023 12:54:26 +0000 Subject: [PATCH 012/116] LINE events working for sys.setrace. --- Python/legacy_tracing.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 016c1d99a35420..187a3246b71616 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ceval.h" #include "pycore_instruments.h" #include "pycore_pyerrors.h" @@ -231,6 +232,9 @@ sys_trace_handled_exception( assert(args[0] == (PyObject *)iframe->f_code); int offset = _PyLong_AsInt(args[1]); assert(offset >= 0); + if (_PyCode_CODE(iframe->f_code)[offset].opcode == END_ASYNC_FOR) { + Py_RETURN_NONE; + } int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); if (line < 0) { Py_RETURN_NONE; @@ -266,6 +270,12 @@ sys_trace_branch_func( /* Backwards jump -- always trace */ tstate->trace_info.code = NULL; } + else { + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (to_line == from_line) { + Py_RETURN_NONE; + } + } return trace_line(tstate, iframe, self, frame, to_line); } From e52cbe668a75be23a75b5485e0f89628eb237866 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jan 2023 11:10:39 +0000 Subject: [PATCH 013/116] Add lots of internal debugging for instrumentation. --- Include/cpython/code.h | 14 +- Include/internal/pycore_interp.h | 1 - Objects/codeobject.c | 6 - Python/bytecodes.c | 14 +- Python/generated_cases.c.h | 14 +- Python/instrumentation.c | 378 ++++++++++++++++++------------- Python/pystate.c | 1 - 7 files changed, 232 insertions(+), 196 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fcad7f1fb6ce22..ffb6d513fcf7fd 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -49,23 +49,13 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; -typedef struct _PyInstrumentationOffsets { - int8_t multi_tools; - int8_t lines; - int8_t instructions; -} _PyInstrumentationOffsets; - -typedef union _PyInstrumentationLayout { - _PyInstrumentationOffsets offsets; - int32_t bits; -} _PyInstrumentationLayout; - typedef struct { uint8_t original_opcode; int8_t line_delta; } _PyCoLineInstrumentationData; typedef struct { + _Py_MonitoringMatrix matrix; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; @@ -75,8 +65,6 @@ typedef struct { typedef struct { uint64_t monitoring_version; /* current instrumentation version */ - _PyInstrumentationLayout layout; - _Py_MonitoringMatrix monitoring_matrix; _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ } _PyCoInstrumentation; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 59a52f706682e8..184db0505864e6 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -191,7 +191,6 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; - _PyInstrumentationLayout instrumentation_layout; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f1eff3c0a1ad79..e96b0ca5d48c88 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -402,12 +402,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) if (_Py_next_func_version != 0) { _Py_next_func_version++; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - co->_co_instrumentation.monitoring_matrix.tools[e] = 0; - } - co->_co_instrumentation.layout.offsets.multi_tools = 0; - co->_co_instrumentation.layout.offsets.lines = 0; - co->_co_instrumentation.layout.offsets.instructions = 0; co->_co_instrumentation.monitoring_data = NULL; co->_co_instrumentation.monitoring_version = 0; /* not set */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ad2240543c2be5..31d1c5d2050004 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1897,7 +1897,7 @@ dummy_func( target = next_instr + 2; } err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); ERROR_IF(err, error); } @@ -3458,7 +3458,7 @@ dummy_func( assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); ERROR_IF(err, error); if (err) { JUMPBY(oparg); @@ -3477,7 +3477,7 @@ dummy_func( assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); ERROR_IF(err, error); if (err == 0) { JUMPBY(oparg); @@ -3496,7 +3496,7 @@ dummy_func( assert(err == 0 || err == 1); int offset = err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3509,7 +3509,7 @@ dummy_func( assert(err == 0 || err == 1); int offset = (1-err)*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3527,7 +3527,7 @@ dummy_func( offset = 0; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3545,7 +3545,7 @@ dummy_func( offset = oparg; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b2ea7dcf8fa484..99f8ba38aa1d72 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2158,7 +2158,7 @@ target = next_instr + 2; } err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); if (err) goto pop_2_error; STACK_SHRINK(2); JUMPBY(2); @@ -3785,7 +3785,7 @@ assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); if (err) goto error; if (err) { JUMPBY(oparg); @@ -3805,7 +3805,7 @@ assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); if (err) goto error; if (err == 0) { JUMPBY(oparg); @@ -3825,7 +3825,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3839,7 +3839,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3858,7 +3858,7 @@ offset = 0; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3877,7 +3877,7 @@ offset = oparg; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6668dc535551aa..9934e03aeed6c5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -106,8 +106,10 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in /* No error checking -- Don't use this for anything but experimental debugging */ static void -dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, int star, FILE*out) +dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; @@ -124,18 +126,18 @@ dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, in else { base_opcode = _PyOpcode_Deopt[opcode]; } - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[base_opcode]; if (i == star) { fprintf(out, "** "); starred = true; } - fprintf(out, "Offset: %d, Instruction: %s: ", i, _PyOpcode_OpName[opcode]); + fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]); dump_instrumentation_data_tools(code, data->tools, i, out); dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); + /* TO DO -- Use instruction length, not cache count */ + i += _PyOpcode_Caches[base_opcode]; } if (!starred && star >= 0) { fprintf(out, "Error offset not at valid instruction offset: %d\n", star); @@ -143,12 +145,171 @@ dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, in dump_instrumentation_data_tools(code, data->tools, star, out); dump_instrumentation_data_lines(code, data->lines, star, out); dump_instrumentation_data_line_tools(code, data->line_tools, star, out); + fprintf(out, "\n"); + } +} + +#define OFFSET_SHIFT 5 + +/* Line delta. + * 8 bit value. + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + */ + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + if (line < 0) { + return -128; + } + // assert(line >= code->co_firstlineno); + // assert(offset >= code->_co_firsttraceable); + // assert(offset < Py_SIZE(code)); + int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); + if (delta < 128 && delta > -128) { + return delta; + } + return -127; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta > -127) { + // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); + return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } +} + + +static inline bool +is_instrumented(int opcode) { + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + +static const int8_t EVENT_FOR_OPCODE[256] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, +}; + +static const bool OPCODE_HAS_EVENT[256] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [INSTRUMENTED_RESUME] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, + [COMPARE_AND_BRANCH] = true, +}; + +bool valid_opcode(int opcode) { + if (opcode > 0 && opcode < 255 && + _PyOpcode_OpName[opcode] && + _PyOpcode_OpName[opcode][0] != '<' + ) { + return true; } + return false; } -_PyInstrumentationOffsets -get_offsets(PyCodeObject *code) { - return code->_co_instrumentation.layout.offsets; +#define CHECK(test) do { \ + if (!(test)) { \ + dump_instrumentation_data(code, i, stderr); \ + } \ + assert(test); \ +} while (0) + +static void +sanity_check_instrumentation(PyCodeObject *code) +{ + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + if (data == NULL) { + return; + } + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- Check INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + assert(data->lines); + assert(valid_opcode(data->lines[i].original_opcode)); + opcode = data->lines[i].original_opcode; + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + CHECK(opcode != INSTRUMENTED_LINE); + } + else if (data->lines) { + CHECK(data->lines[i].original_opcode == 0 || + data->lines[i].original_opcode == 255); + } + if (is_instrumented(opcode)) { + int deinstrumented = DE_INSTRUMENT[opcode]; + CHECK(valid_opcode(deinstrumented)); + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + if (_PyOpcode_Caches[deinstrumented]) { + CHECK(instr[1].cache > 0); + } + opcode = deinstrumented; + } + if (data->lines && opcode != END_FOR) { + int line1 = compute_line(code, i, data->lines[i].line_delta); + int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + CHECK(line1 == line2); + } + CHECK(valid_opcode(opcode)); + opcode = _PyOpcode_Deopt[opcode]; + CHECK(valid_opcode(opcode)); + if (data->tools && OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + uint8_t local_tools = data->tools[i]; + uint8_t global_tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + i += _PyOpcode_Caches[opcode]; + } } static inline uint8_t @@ -162,18 +323,13 @@ get_tools(PyCodeObject * code, int index, int event) tools = monitoring->tools[index]; } else { - tools = code->_co_instrumentation.monitoring_matrix.tools[event]; + tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; } assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); return tools; } -static inline bool -is_instrumented(int opcode) { - return INSTRUMENTED_OPCODES[opcode] == opcode; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -295,7 +451,7 @@ instrument_line(PyCodeObject *code, int offset) } _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; if (lines->original_opcode != 255) { - dump_instrumentation_data(code, code->_co_instrumentation.monitoring_data, offset, stderr); + dump_instrumentation_data(code, offset, stderr); } assert(lines->original_opcode == 255); if (is_instrumented(opcode)) { @@ -327,7 +483,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); @@ -338,9 +494,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { - assert (code->_co_instrumentation.layout.offsets.lines >= 0); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->line_tools) + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->line_tools) { uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; *toolsptr &= ~tools; @@ -350,7 +505,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -364,6 +519,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data && code->_co_instrumentation.monitoring_data->tools ) { @@ -382,10 +538,9 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert (get_offsets(code).lines >= 0); assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->line_tools + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->line_tools ) { code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; } @@ -448,7 +603,7 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); /* No per-instruction monitoring yet */ - assert(!code->_co_instrumentation.layout.offsets.instructions); + assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -495,7 +650,7 @@ _Py_call_instrumentation( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; @@ -511,7 +666,7 @@ _Py_call_instrumentation_arg( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -554,7 +709,7 @@ _Py_call_instrumentation_exc( _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -572,49 +727,6 @@ _Py_call_instrumentation_exc( Py_DECREF(instruction_offset_obj); } -/* Line delta. - * 8 bit value. - * if line_delta == -128: - * line = None # represented as -1 - * elif line == -127: - * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - * else: - * line = first_line + (offset >> 3) + line_delta; - */ - -static int8_t -compute_line_delta(PyCodeObject *code, int offset, int line) -{ - if (line < 0) { - return -128; - } - assert(line >= code->co_firstlineno); - assert(offset >= code->_co_firsttraceable); - assert(offset < Py_SIZE(code)); - int delta = line - code->co_firstlineno - (offset >> 3); - if (delta < 128 && delta > -128) { - return delta; - } - return -127; -} - -static int -compute_line(PyCodeObject *code, int offset, int8_t line_delta) -{ - if (line_delta > -127) { - assert((offset >> 3) + line_delta >= 0); - return code->co_firstlineno + (offset >> 3) + line_delta; - } - if (line_delta == -128) { - return -1; - } - else { - assert(line_delta == -127); - /* Compute from table */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } -} - int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { @@ -633,7 +745,7 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) { assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; @@ -690,49 +802,6 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) return callback; } -static const int8_t EVENT_FOR_OPCODE[256] = { - [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [CALL] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, - [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [RESUME] = -1, - [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, -}; - -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [JUMP_IF_FALSE_OR_POP] = true, - [JUMP_IF_TRUE_OR_POP] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, -}; - static inline _Py_MonitoringMatrix matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { @@ -764,6 +833,17 @@ matrix_empty(_Py_MonitoringMatrix m) return true; } +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + static inline void matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) { @@ -786,15 +866,6 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -static inline _PyInstrumentationLayout -layout_from_flags(bool multitools, bool lines, bool instrs) { - _PyInstrumentationLayout result; - result.offsets.multi_tools = multitools; - result.offsets.lines = lines; - result.offsets.instructions = instrs; - return result; -} - static inline int tools_data_offset() { return 0; @@ -830,7 +901,7 @@ initialize_tools(PyCodeObject *code) assert(event > 0); } assert(event >= 0); - tools[i] = code->_co_instrumentation.monitoring_matrix.tools[event]; + tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; assert(tools[i] != 0); } else { @@ -843,18 +914,13 @@ initialize_tools(PyCodeObject *code) static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { + int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ if (opcode == END_FOR) { + bounds->ar_line = -1; return false; } - while (bounds->ar_end <= index*(int)sizeof(_Py_CODEUNIT)) { - if (!_PyLineTable_NextAddressRange(bounds)) { - return false; - } - } - assert(index*(int)sizeof(_Py_CODEUNIT) >= bounds->ar_start && - index*(int)sizeof(_Py_CODEUNIT) < bounds->ar_end); - return bounds->ar_line >= 0 && bounds->ar_line != current_line; + return line >= 0 && line != current_line; } static void @@ -865,12 +931,12 @@ initialize_lines(PyCodeObject *code) int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; _PyCode_InitAddressRange(code, &range); - for (int i = 0; i <= code->_co_firsttraceable && i < code_len; i++) { + for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { line_data[i].original_opcode = 0; line_data[i].line_delta = -128; } int current_line = -1; - for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); if (is_new_line(i, current_line, &range, opcode)) { line_data[i].original_opcode = 255; @@ -880,7 +946,7 @@ initialize_lines(PyCodeObject *code) /* Mark as not being an instrumentation point */ line_data[i].original_opcode = 0; } - assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; @@ -904,7 +970,6 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { - _PyInstrumentationLayout new_layout = interp->instrumentation_layout; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; int code_len = (int)Py_SIZE(code); if (code->_co_instrumentation.monitoring_data == NULL) { @@ -913,13 +978,15 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } + code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->tools == NULL) { + bool multitools = multiple_tools(interp->monitoring_matrix); + if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { PyErr_NoMemory(); @@ -931,7 +998,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_tools(code); } } - if (new_layout.offsets.lines) { + if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -940,7 +1007,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_lines(code); } - if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { + if (multitools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { PyErr_NoMemory(); @@ -949,8 +1016,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } - assert(new_layout.offsets.instructions == 0); - code->_co_instrumentation.layout.bits = new_layout.bits; return 0; } @@ -973,35 +1038,33 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { if (is_instrumentation_up_to_date(code, interp)) { - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix)); + assert(interp->monitoring_version == 0 || + matrix_equals(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix) + ); return 0; } bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); - _PyInstrumentationLayout new_layout = interp->instrumentation_layout; - _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; - if (new_layout.bits != old_layout.bits) { - if (update_instrumentation_data(code, interp)) { - return -1; - } + if (update_instrumentation_data(code, interp)) { + return -1; } _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_matrix; + removed_events = code->_co_instrumentation.monitoring_data->matrix; new_events = interp->monitoring_matrix; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix); - new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_matrix); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix); + new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_data->matrix); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_matrix = interp->monitoring_matrix; - assert(code->_co_instrumentation.layout.bits == interp->instrumentation_layout.bits); + code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { + sanity_check_instrumentation(code); return 0; } /* Insert instrumentation */ @@ -1069,6 +1132,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } + sanity_check_instrumentation(code); return 0; } @@ -1104,20 +1168,12 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (existing_events == events) { return; } - bool multitools = false; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); - uint8_t tools = interp->monitoring_matrix.tools[e]; - if (_Py_popcount32(tools) > 1) { - multitools = true; - } } interp->monitoring_version++; - bool line_monitoring = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); - bool instruction_monitoring = false; - interp->instrumentation_layout = layout_from_flags(multitools, line_monitoring, instruction_monitoring); instrument_all_executing_code_objects(interp); } diff --git a/Python/pystate.c b/Python/pystate.c index f76ef37c70c3eb..8e39759b8ae9a9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -315,7 +315,6 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } - interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ 0, 0, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 6c8be7e0217af4576859b1c435661ad3170147c8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jan 2023 21:38:39 +0000 Subject: [PATCH 014/116] Add more tests. Get those and some other passing. --- Lib/test/test_monitoring.py | 59 ++++- Objects/codeobject.c | 2 +- Python/instrumentation.c | 448 +++++++++++++++++++++--------------- Python/legacy_tracing.c | 6 + 4 files changed, 324 insertions(+), 191 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index bab69d55446dcc..fabf3ab8ae6d05 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -8,6 +8,8 @@ import types import collections +PAIR = (0,1) + def f1(): pass @@ -15,6 +17,10 @@ def f2(): len([]) sys.getsizeof(0) +def floop(): + for item in PAIR: + pass + def gen(): yield yield @@ -162,6 +168,7 @@ def caught(): "start", "raise", "exception_handled", + "branch", "return", ] @@ -311,7 +318,6 @@ def __init__(self, events): def __call__(self, code, event): self.events.append(event) - print(len(self.events)) if self.disable: return sys.monitoring.DISABLE @@ -481,16 +487,61 @@ def test_lines_single(self): try: self.assertEqual(sys.monitoring._all_events(), {}) events = [] - counter = RecorderWithDisable(events) - sys.monitoring.register_callback(TEST_TOOL, E.LINE, counter) + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) sys.monitoring.set_events(TEST_TOOL, E.LINE) f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [487, 12, 488]) + self.assertEqual(events, [493, 14, 494]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_loop(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + floop() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(events, [510, 21, 22, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() + def test_lines_two(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + events2 = [] + recorder2 = RecorderWithDisable(events2) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, recorder2) + sys.monitoring.set_events(TEST_TOOL, E.LINE); sys.monitoring.set_events(TEST_TOOL2, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + expected = [530, 14, 531] + self.assertEqual(events, expected) + self.assertEqual(events2, expected) + finally: + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + + + diff --git a/Objects/codeobject.c b/Objects/codeobject.c index e96b0ca5d48c88..535a521e3170b0 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1849,7 +1849,7 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _PyOpcode_Deopt[_Py_OPCODE(_PyCode_CODE(co)[i])]; + int deop = _Py_GetBaseOpcode(co, i); SCRAMBLE_IN(deop); SCRAMBLE_IN(_Py_OPARG(_PyCode_CODE(co)[i])); i += _PyOpcode_Caches[deop]; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9934e03aeed6c5..797df8ebc315cf 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,6 +14,67 @@ static PyObject DISABLE = &PyBaseObject_Type }; +static const int8_t EVENT_FOR_OPCODE[256] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, +}; + +static const bool OPCODE_HAS_EVENT[256] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [INSTRUMENTED_RESUME] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, + [COMPARE_AND_BRANCH] = true, + [INSTRUMENTED_JUMP_FORWARD] = true, + [INSTRUMENTED_JUMP_BACKWARD] = true, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = true, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = true, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, + [INSTRUMENTED_POP_JUMP_IF_NONE] = true, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, + [INSTRUMENTED_COMPARE_AND_BRANCH] = true, +}; + static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, @@ -21,7 +82,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, - [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, @@ -103,6 +164,14 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } } +static void +dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) +{ + fprintf(out, "%s matrix:\n", prefix); + for (int event = 0; event < PY_MONITORING_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, matrix.tools[event]); + } +} /* No error checking -- Don't use this for anything but experimental debugging */ static void @@ -114,6 +183,8 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) fprintf(out, "NULL\n"); return; } + dump_matrix("Global", PyInterpreterState_Get()->monitoring_matrix, out); + dump_matrix("Code", data->matrix, out); int code_len = (int)Py_SIZE(code); bool starred = false; for (int i = 0; i < code_len; i++) { @@ -149,7 +220,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } } -#define OFFSET_SHIFT 5 +#define OFFSET_SHIFT 4 /* Line delta. * 8 bit value. @@ -200,49 +271,6 @@ is_instrumented(int opcode) { return INSTRUMENTED_OPCODES[opcode] == opcode; } -static const int8_t EVENT_FOR_OPCODE[256] = { - [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [CALL] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, - [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [RESUME] = -1, - [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, -}; - -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [JUMP_IF_FALSE_OR_POP] = true, - [JUMP_IF_TRUE_OR_POP] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, -}; - bool valid_opcode(int opcode) { if (opcode > 0 && opcode < 255 && _PyOpcode_OpName[opcode] && @@ -253,6 +281,81 @@ bool valid_opcode(int opcode) { return false; } +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } + } + return true; +} + +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(val == 0 || val == 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + #define CHECK(test) do { \ if (!(test)) { \ dump_instrumentation_data(code, i, stderr); \ @@ -267,15 +370,23 @@ sanity_check_instrumentation(PyCodeObject *code) if (data == NULL) { return; } + _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; + assert(matrix_equals( + code->_co_instrumentation.monitoring_data->matrix, + tools_matrix) + ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; /* TO DO -- Check INSTRUMENTED_OPCODE */ if (opcode == INSTRUMENTED_LINE) { - assert(data->lines); - assert(valid_opcode(data->lines[i].original_opcode)); + CHECK(data->lines); + CHECK(valid_opcode(data->lines[i].original_opcode)); opcode = data->lines[i].original_opcode; + CHECK(opcode != END_FOR); + CHECK(opcode != RESUME); + CHECK(opcode != INSTRUMENTED_RESUME); if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } @@ -293,6 +404,12 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(instr[1].cache > 0); } opcode = deinstrumented; + int event = EVENT_FOR_OPCODE[opcode]; + if (event < 0) { + /* RESUME fixup */ + event = instr->oparg; + } + CHECK(tools_matrix.tools[event] != 0); } if (data->lines && opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); @@ -302,34 +419,25 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(valid_opcode(opcode)); opcode = _PyOpcode_Deopt[opcode]; CHECK(valid_opcode(opcode)); - if (data->tools && OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; + if (data->tools) { uint8_t local_tools = data->tools[i]; - uint8_t global_tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); + if (OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + if (event == -1) { + /* RESUME fixup */ + event = instr->oparg; + } + uint8_t global_tools = tools_matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + else { + CHECK(local_tools == 0xff); + } } i += _PyOpcode_Caches[opcode]; } } -static inline uint8_t -get_tools(PyCodeObject * code, int index, int event) -{ - uint8_t tools; - assert(event != PY_MONITORING_EVENT_LINE); - assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (monitoring && monitoring->tools) { - tools = monitoring->tools[index]; - } - else { - tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - } - assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); - return tools; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -366,8 +474,7 @@ de_instrument(PyCodeObject *code, int offset, int event) int deinstrumented = DE_INSTRUMENT[opcode]; int base_opcode; if (deinstrumented) { - base_opcode = _PyOpcode_Deopt[deinstrumented]; - assert(base_opcode != 0); + base_opcode = deinstrumented; instr->opcode = base_opcode; } else { @@ -375,11 +482,18 @@ de_instrument(PyCodeObject *code, int offset, int event) assert(opcode == INSTRUMENTED_LINE); _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; int orignal_opcode = lines->original_opcode; - base_opcode = _PyOpcode_Deopt[orignal_opcode]; - assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); - assert(base_opcode != 0); - lines->original_opcode = base_opcode; + if (is_instrumented(orignal_opcode)) { + base_opcode = DE_INSTRUMENT[orignal_opcode]; + lines->original_opcode = DE_INSTRUMENT[orignal_opcode]; + assert(lines->original_opcode != 0); + + } + else { + base_opcode = orignal_opcode; + } } + assert(base_opcode != 0); + assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -428,9 +542,6 @@ instrument(PyCodeObject *code, int offset) else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; - if (!instrumented) { - printf("OPCODE: %s\n", _PyOpcode_OpName[opcode]); - } assert(instrumented); instr->opcode = instrumented; /* TO DO -- Use instruction length, not cache count */ @@ -469,11 +580,27 @@ instrument_line(PyCodeObject *code, int offset) assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); } +#ifndef NDEBUG +static bool +instruction_has_event(PyCodeObject *code, int offset) +{ + _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; + int opcode = instr.opcode; + /* TO DO -- Handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); + return OPCODE_HAS_EVENT[opcode]; +} +#endif + static void remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(instruction_has_event(code, offset)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; @@ -592,10 +719,29 @@ static inline int most_significant_bit(uint8_t bits) { } } +static inline uint8_t +get_tools(PyCodeObject * code, int index, int event) +{ + uint8_t tools; + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + tools = monitoring->tools[index]; + } + else { + tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + } + assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + return tools; +} + static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } @@ -626,17 +772,6 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (a.tools[i] != b.tools[i]) { - return false; - } - } - return true; -} - static bool is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { @@ -802,75 +937,6 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) return callback; } -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & ~b.tools[i]; - } - return res; -} - -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & b.tools[i]; - } - return res; -} - -static inline bool -matrix_empty(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (m.tools[i]) { - return false; - } - } - return true; -} - -static inline int -multiple_tools(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { - return true; - } - } - return false; -} - -static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); -} - -static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) -{ - _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - return result; -} - -static inline int -tools_data_offset() { - return 0; -} - static void initialize_tools(PyCodeObject *code) { @@ -879,7 +945,7 @@ initialize_tools(PyCodeObject *code) int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); + int opcode = instr->opcode; if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; } @@ -893,8 +959,7 @@ initialize_tools(PyCodeObject *code) if (instrumented) { int8_t event; if (opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg != 0; + event = instr->oparg != 0; } else { event = EVENT_FOR_OPCODE[opcode]; @@ -902,12 +967,21 @@ initialize_tools(PyCodeObject *code) } assert(event >= 0); tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - assert(tools[i] != 0); + CHECK(tools[i] != 0); } else { tools[i] = 0; } } +#ifdef Py_DEBUG + /* Initialize tools for invalid locations to all ones to try to catch errors */ + else { + tools[i] = 0xff; + } + for (int j = 1; j <= _PyOpcode_Caches[opcode]; j++) { + tools[i+j] = 0xff; + } +#endif i += _PyOpcode_Caches[opcode]; } } @@ -915,11 +989,6 @@ initialize_tools(PyCodeObject *code) static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); - /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ - if (opcode == END_FOR) { - bounds->ar_line = -1; - return false; - } return line >= 0 && line != current_line; } @@ -933,21 +1002,29 @@ initialize_lines(PyCodeObject *code) _PyCode_InitAddressRange(code, &range); for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { line_data[i].original_opcode = 0; - line_data[i].line_delta = -128; + line_data[i].line_delta = -127; } int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = 255; - current_line = range.ar_line; + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ + if (opcode == END_FOR || opcode == RESUME) { + line_data[i].original_opcode = 0; + line_data[i].line_delta = -127; } else { - /* Mark as not being an instrumentation point */ - line_data[i].original_opcode = 0; + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = 255; + current_line = range.ar_line; + } + else { + /* Mark as not being an instrumentation point */ + line_data[i].original_opcode = 0; + } + // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); } - // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; @@ -970,7 +1047,6 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; int code_len = (int)Py_SIZE(code); if (code->_co_instrumentation.monitoring_data == NULL) { code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); @@ -978,7 +1054,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; + code->_co_instrumentation.monitoring_data->matrix = (_Py_MonitoringMatrix){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; @@ -992,11 +1068,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - if (restarted) { - memset(code->_co_instrumentation.monitoring_data->tools, 0, code_len); - } else { - initialize_tools(code); - } + initialize_tools(code); } if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1043,7 +1115,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); if (update_instrumentation_data(code, interp)) { @@ -1052,6 +1123,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { removed_events = code->_co_instrumentation.monitoring_data->matrix; new_events = interp->monitoring_matrix; @@ -1064,13 +1136,17 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { - sanity_check_instrumentation(code); + //sanity_check_instrumentation(code); return 0; } /* Insert instrumentation */ for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + /* TO DO handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + } int base_opcode; if (DE_INSTRUMENT[opcode]) { base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; @@ -1114,7 +1190,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools || removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; - for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1132,7 +1208,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - sanity_check_instrumentation(code); + //sanity_check_instrumentation(code); return 0; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 187a3246b71616..5bef8ab9e69ea4 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -259,6 +259,12 @@ sys_trace_branch_func( _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } int from = _PyLong_AsInt(args[1]); assert(from >= 0); int to = _PyLong_AsInt(args[2]); From c9e1e21ab4694df3b01d6d43b4e2a48f3b205a26 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 19 Jan 2023 11:45:09 +0000 Subject: [PATCH 015/116] Fix LINE instrumentation and frame.set_lineno support (mostly) --- Include/cpython/pystate.h | 2 +- Include/internal/pycore_instruments.h | 4 +- Lib/test/test_monitoring.py | 2 +- Objects/frameobject.c | 54 ++++++++++++------- Python/bytecodes.c | 18 ++++++- Python/generated_cases.c.h | 18 ++++++- Python/instrumentation.c | 76 +++++++++++++++++++++------ Python/legacy_tracing.c | 52 +++++++----------- Python/pystate.c | 1 + 9 files changed, 153 insertions(+), 74 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 84a22d2f2f1abd..0c70cbb1b05183 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -129,7 +129,7 @@ struct _ts { This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; - int tracing_what; /* The event currently being traced, if any. */ + int what_event; /* The event currently being monitored, if any. */ uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 98cba9884b4f60..33c894368d2cac 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -67,8 +67,8 @@ _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); extern int -_Py_call_instrumentation_line(PyThreadState *tstate, - PyCodeObject *code, _Py_CODEUNIT *instr); +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, + _Py_CODEUNIT *instr); int _Py_call_instrumentation_jump( diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fabf3ab8ae6d05..1e5ef85eb4dcab 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 21, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d05b0b7d65e029..6f3f90f13c662e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -686,32 +686,48 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore * In addition, jumps are forbidden when not tracing, * as this is a debugging feature. */ - switch(PyThreadState_GET()->tracing_what) { - case PyTrace_EXCEPTION: + int what_event = PyThreadState_GET()->what_event; + if (what_event < 0) { + PyErr_Format(PyExc_ValueError, + "f_lineno can only be set by a debugger"); + return -1; + } + switch(what_event) { + case PY_MONITORING_EVENT_PY_START: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' event of a new frame"); + return -1; + case PY_MONITORING_EVENT_PY_RESUME: + break; + case PY_MONITORING_EVENT_PY_RETURN: PyErr_SetString(PyExc_ValueError, - "can only jump from a 'line' trace event"); + "can't jump from a 'return' event"); return -1; - case PyTrace_CALL: - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); + case PY_MONITORING_EVENT_PY_YIELD: + break; + case PY_MONITORING_EVENT_CALL: + case PY_MONITORING_EVENT_C_RETURN: + PyErr_SetString(PyExc_ValueError, + "can't jump during a call"); return -1; - case PyTrace_LINE: + case PY_MONITORING_EVENT_LINE: break; - case PyTrace_RETURN: - if (state == FRAME_SUSPENDED) { - break; - } - /* fall through */ + case PY_MONITORING_EVENT_PY_UNWIND: + case PY_MONITORING_EVENT_PY_THROW: + case PY_MONITORING_EVENT_RAISE: + case PY_MONITORING_EVENT_C_RAISE: + case PY_MONITORING_EVENT_JUMP: + case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_INSTRUCTION: + case PY_MONITORING_EVENT_EXCEPTION_HANDLED: + PyErr_Format(PyExc_ValueError, + "can only jump from 'line' event"); + return -1; default: - PyErr_SetString(PyExc_ValueError, - "can only jump from a 'line' trace event"); + PyErr_SetString(PyExc_SystemError, + "unexpected event type"); return -1; } - if (!f->f_trace) { - PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a trace function"); - return -1; - } int new_lineno; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31d1c5d2050004..dfd5b7bf304263 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -127,9 +127,17 @@ dummy_func( next_instr--; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg != 0, frame, next_instr-1); ERROR_IF(err, error); + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -763,10 +771,18 @@ dummy_func( inst(INSTRUMENTED_YIELD_VALUE, ( -- )) { PyObject *val = TOP(); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); ERROR_IF(err, error); + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -3412,7 +3428,7 @@ dummy_func( inst(INSTRUMENTED_LINE, ( -- )) { _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame->f_code, next_instr-1); + tstate, frame, next_instr-1); ERROR_IF(original_opcode < 0, error); next_instr--; if (frame->prev_instr != next_instr) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 99f8ba38aa1d72..7a2414958ca1b5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -16,9 +16,17 @@ next_instr--; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg != 0, frame, next_instr-1); if (err) goto error; + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -928,10 +936,18 @@ TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *val = TOP(); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); if (err) goto error; + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -3739,7 +3755,7 @@ TARGET(INSTRUMENTED_LINE) { _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame->f_code, next_instr-1); + tstate, frame, next_instr-1); if (original_opcode < 0) goto error; next_instr--; if (frame->prev_instr != next_instr) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 797df8ebc315cf..09802dbcf3025a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -690,9 +690,12 @@ call_one_instrument( if (instrument == NULL) { return 0; } + int old_what = tstate->what_event; + tstate->what_event = event; tstate->tracing++; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); tstate->tracing--; + tstate->what_event = old_what; if (res == NULL) { return -1; } @@ -877,8 +880,9 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) } int -_Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { + PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); @@ -891,6 +895,11 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); + if (tstate->trace_info.code == frame->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ + return original_opcode; + } uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[offset] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; @@ -920,6 +929,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD remove_line_tools(code, offset, 1 << tool); } } + tstate->trace_info.code = frame->f_code; + tstate->trace_info.line = line; Py_DECREF(line_obj); assert(original_opcode != 0); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -992,6 +1003,8 @@ static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, return line >= 0 && line != current_line; } +#define LINE_MARKER 255 + static void initialize_lines(PyCodeObject *code) { @@ -1005,31 +1018,62 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; + int extended_arg = 0; + _Py_CODEUNIT *bytecode = _PyCode_CODE(code); + for (int i = code->_co_firsttraceable; i < code_len; i++) { + line_data[i].original_opcode = 0; + } for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); - /* END_FOR cannot start a line, as it is skipped by FOR_ITER - * RESUME must not be instrumented with INSTRUMENT_LINE */ - if (opcode == END_FOR || opcode == RESUME) { - line_data[i].original_opcode = 0; - line_data[i].line_delta = -127; - } - else { - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = 255; - current_line = range.ar_line; - } - else { - /* Mark as not being an instrumentation point */ + switch (opcode) { + case END_FOR: + case RESUME: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; + line_data[i].line_delta = -127; + extended_arg = 0; + continue; + case EXTENDED_ARG: + extended_arg = (extended_arg << 8) + bytecode[i].oparg; + line_data[i].line_delta = -127; + continue; + case JUMP_IF_FALSE_OR_POP: + case JUMP_IF_TRUE_OR_POP: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case JUMP_FORWARD: + /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ + { + int offset = (extended_arg << 8) + bytecode[i].oparg; + line_data[i+1+offset].original_opcode = LINE_MARKER; + break; } - // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); + case FOR_ITER: + { + /* Skip over END_FOR */ + int offset = (extended_arg << 8) + bytecode[i].oparg + 1; + line_data[i+1+offset].original_opcode = LINE_MARKER; + break; + } + case JUMP_BACKWARD: + { + int offset = (extended_arg << 8) + bytecode[i].oparg; + line_data[i+1-offset].original_opcode = LINE_MARKER; + break; + } + } + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = LINE_MARKER; + current_line = range.ar_line; } + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; line_data[i].line_delta = -128; } + extended_arg = 0; } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 5bef8ab9e69ea4..d232917273f8d0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -163,14 +163,8 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } Py_INCREF(frame); - tstate->trace_info.code = iframe->f_code; - tstate->trace_info.line = frame->f_lineno = line; + frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); frame->f_lineno = 0; Py_DECREF(frame); @@ -180,7 +174,6 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, Py_RETURN_NONE; } - static PyObject * sys_trace_line_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -245,7 +238,7 @@ sys_trace_handled_exception( static PyObject * -sys_trace_branch_func( +sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -256,6 +249,19 @@ sys_trace_branch_func( } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); + int from = _PyLong_AsInt(args[1]); + assert(from >= 0); + int to = _PyLong_AsInt(args[2]); + assert(to >= 0); + /* Forward jumps should have been handled by the line instrumentation. + * We need to handle backward jumps, as the semantics differ. + * PEP 669 only generates events when a new line is seen. + * Legacy tracing generates events for all backward jumps even on + * the same line */ + if (to > from) { + /* Forwards jump */ + Py_RETURN_NONE; + } _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); @@ -265,23 +271,9 @@ sys_trace_branch_func( if (!frame->f_trace_lines) { Py_RETURN_NONE; } - int from = _PyLong_AsInt(args[1]); - assert(from >= 0); - int to = _PyLong_AsInt(args[2]); - assert(to >= 0); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); - if (from > to) { - /* Backwards jump -- always trace */ - tstate->trace_info.code = NULL; - } - else { - int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); - if (to_line == from_line) { - Py_RETURN_NONE; - } - } return trace_line(tstate, iframe, self, frame, to_line); } @@ -459,8 +451,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -468,11 +460,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_handled_exception, PyTrace_LINE, - PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { - return -1; - } /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } if (tstate->interp->sys_tracing_threads) { @@ -480,9 +467,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW)); + (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); /* TO DO -- opcode events */ if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); diff --git a/Python/pystate.c b/Python/pystate.c index 8e39759b8ae9a9..30a30360778629 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -830,6 +830,7 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->monitoring = 0; + tstate->what_event = -1; tstate->_initialized = 1; } From e52e8d32812bc4eb097fa1d3a3cc7c26ebf579fc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 10:02:24 +0000 Subject: [PATCH 016/116] Refining line event generation. --- Include/internal/pycore_instruments.h | 4 + Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 + Lib/opcode.py | 2 +- Objects/frameobject.c | 14 +- Python/bytecodes.c | 14 + Python/generated_cases.c.h | 14 + Python/instrumentation.c | 676 ++++++++++++++++++-------- Python/legacy_tracing.c | 67 +-- Python/opcode_metadata.h | 1 + Python/opcode_targets.h | 2 +- 11 files changed, 530 insertions(+), 269 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 33c894368d2cac..6c76027889457d 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,6 +70,10 @@ extern int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); +extern int +_Py_call_instrumentation_instruction( + PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); + int _Py_call_instrumentation_jump( PyThreadState *tstate, int event, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 05a9d2e4c4673b..fb1359a93b8325 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -481,7 +482,7 @@ static const char *const _PyOpcode_OpName[263] = { [235] = "<235>", [236] = "<236>", [237] = "<237>", - [238] = "<238>", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ - case 238: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index e5adadf8fcb7d0..88ebea7c0efc20 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define INSTRUMENTED_INSTRUCTION 238 #define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 diff --git a/Lib/opcode.py b/Lib/opcode.py index 254adef46cc837..7d4091ed1178bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,7 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions +def_op('INSTRUMENTED_INSTRUCTION', 238) def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) @@ -236,7 +237,6 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 252) def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 253) - def_op('INSTRUMENTED_LINE', 254) # 255 is reserved. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6f3f90f13c662e..dc196846d57ab4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -689,20 +689,17 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore int what_event = PyThreadState_GET()->what_event; if (what_event < 0) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a debugger"); + "f_lineno can only be set in a trace function"); return -1; } switch(what_event) { case PY_MONITORING_EVENT_PY_START: PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' event of a new frame"); + "can't jump from the 'call' trace event of a new frame"); return -1; case PY_MONITORING_EVENT_PY_RESUME: break; - case PY_MONITORING_EVENT_PY_RETURN: - PyErr_SetString(PyExc_ValueError, - "can't jump from a 'return' event"); - return -1; + case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: break; case PY_MONITORING_EVENT_CALL: @@ -710,8 +707,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore PyErr_SetString(PyExc_ValueError, "can't jump during a call"); return -1; - case PY_MONITORING_EVENT_LINE: - break; + case PY_MONITORING_EVENT_PY_RETURN: case PY_MONITORING_EVENT_PY_UNWIND: case PY_MONITORING_EVENT_PY_THROW: case PY_MONITORING_EVENT_RAISE: @@ -721,7 +717,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore case PY_MONITORING_EVENT_INSTRUCTION: case PY_MONITORING_EVENT_EXCEPTION_HANDLED: PyErr_Format(PyExc_ValueError, - "can only jump from 'line' event"); + "can only jump from a 'line' trace event"); return -1; default: PyErr_SetString(PyExc_SystemError, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dfd5b7bf304263..4b3b50bee3f4c4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3448,6 +3448,20 @@ dummy_func( DISPATCH_GOTO(); } + inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + ERROR_IF(next_opcode < 0, error); + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { int err = _Py_call_instrumentation_jump( diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7a2414958ca1b5..317485009c8193 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3775,6 +3775,20 @@ DISPATCH_GOTO(); } + TARGET(INSTRUMENTED_INSTRUCTION) { + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + if (next_opcode < 0) goto error; + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + TARGET(INSTRUMENTED_JUMP_FORWARD) { int err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 09802dbcf3025a..0ade817707bc48 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -123,8 +123,11 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +#ifdef INSTRUMENT_DEBUG + static void dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out) { @@ -164,6 +167,25 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } } +static void +dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentationData *data, int i, FILE*out) +{ + if (data->per_instruction_opcodes == NULL) { + fprintf(out, ", per-inst opcode = NULL"); + } + else { + fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]); + } + if (data->per_instruction_tools == NULL) { + fprintf(out, ", per-inst tools = NULL"); + } + else { + fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]); + } +} + + + static void dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) { @@ -173,6 +195,38 @@ dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) } } +/* Like _Py_GetBaseOpcode but without asserts. + * Does its best to give the right answer, but won't abort + * if something is wrong */ +int get_base_opcode_best_attempt(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (INSTRUMENTED_OPCODES[opcode] != opcode) { + /* Not instrumented */ + return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] == 0) { + return opcode; + } + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + if (code->_co_instrumentation.monitoring_data->lines[offset].original_opcode == 0) { + return opcode; + } + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + if (_PyOpcode_Deopt[opcode] == 0) { + return opcode; + } + return _PyOpcode_Deopt[opcode]; +} + /* No error checking -- Don't use this for anything but experimental debugging */ static void dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) @@ -190,13 +244,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; - int base_opcode; - if (DE_INSTRUMENT[opcode]) { - base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - } - else { - base_opcode = _PyOpcode_Deopt[opcode]; - } if (i == star) { fprintf(out, "** "); starred = true; @@ -205,9 +252,11 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_tools(code, data->tools, i, out); dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); + dump_instrumentation_data_per_instruction(code, data, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); /* TO DO -- Use instruction length, not cache count */ + int base_opcode = get_base_opcode_best_attempt(code, i); i += _PyOpcode_Caches[base_opcode]; } if (!starred && star >= 0) { @@ -216,10 +265,108 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_tools(code, data->tools, star, out); dump_instrumentation_data_lines(code, data->lines, star, out); dump_instrumentation_data_line_tools(code, data->line_tools, star, out); + dump_instrumentation_data_per_instruction(code, data, star, out); fprintf(out, "\n"); } } +#define CHECK(test) do { \ + if (!(test)) { \ + dump_instrumentation_data(code, i, stderr); \ + } \ + assert(test); \ +} while (0) + +bool valid_opcode(int opcode) { + if (opcode > 0 && opcode < 255 && + _PyOpcode_OpName[opcode] && + _PyOpcode_OpName[opcode][0] != '<' + ) { + return true; + } + return false; +} + +static void +sanity_check_instrumentation(PyCodeObject *code) +{ + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + if (data == NULL) { + return; + } + _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; + assert(matrix_equals( + code->_co_instrumentation.monitoring_data->matrix, + tools_matrix) + ); + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- Check INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + CHECK(data->lines); + CHECK(valid_opcode(data->lines[i].original_opcode)); + opcode = data->lines[i].original_opcode; + CHECK(opcode != END_FOR); + CHECK(opcode != RESUME); + CHECK(opcode != INSTRUMENTED_RESUME); + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + CHECK(opcode != INSTRUMENTED_LINE); + } + else if (data->lines) { + CHECK(data->lines[i].original_opcode == 0 || + data->lines[i].original_opcode == 255); + } + if (is_instrumented(opcode)) { + int deinstrumented = DE_INSTRUMENT[opcode]; + CHECK(valid_opcode(deinstrumented)); + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + if (_PyOpcode_Caches[deinstrumented]) { + CHECK(instr[1].cache > 0); + } + opcode = deinstrumented; + int event = EVENT_FOR_OPCODE[opcode]; + if (event < 0) { + /* RESUME fixup */ + event = instr->oparg; + } + CHECK(tools_matrix.tools[event] != 0); + } + if (data->lines && opcode != END_FOR) { + int line1 = compute_line(code, i, data->lines[i].line_delta); + int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + CHECK(line1 == line2); + } + CHECK(valid_opcode(opcode)); + opcode = _PyOpcode_Deopt[opcode]; + CHECK(valid_opcode(opcode)); + if (data->tools) { + uint8_t local_tools = data->tools[i]; + if (OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + if (event == -1) { + /* RESUME fixup */ + event = instr->oparg; + } + uint8_t global_tools = tools_matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + else { + CHECK(local_tools == 0xff); + } + } + i += _PyOpcode_Caches[opcode]; + } +} +#else + +#define CHECK(test) assert(test) + +#endif + #define OFFSET_SHIFT 4 /* Line delta. @@ -268,19 +415,10 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) static inline bool is_instrumented(int opcode) { + assert(opcode != 0); return INSTRUMENTED_OPCODES[opcode] == opcode; } -bool valid_opcode(int opcode) { - if (opcode > 0 && opcode < 255 && - _PyOpcode_OpName[opcode] && - _PyOpcode_OpName[opcode][0] != '<' - ) { - return true; - } - return false; -} - static inline bool matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { @@ -356,142 +494,66 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -#define CHECK(test) do { \ - if (!(test)) { \ - dump_instrumentation_data(code, i, stderr); \ - } \ - assert(test); \ -} while (0) - -static void -sanity_check_instrumentation(PyCodeObject *code) -{ - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; - if (data == NULL) { - return; - } - _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; - assert(matrix_equals( - code->_co_instrumentation.monitoring_data->matrix, - tools_matrix) - ); - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->opcode; - /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { - CHECK(data->lines); - CHECK(valid_opcode(data->lines[i].original_opcode)); - opcode = data->lines[i].original_opcode; - CHECK(opcode != END_FOR); - CHECK(opcode != RESUME); - CHECK(opcode != INSTRUMENTED_RESUME); - if (!is_instrumented(opcode)) { - CHECK(_PyOpcode_Deopt[opcode] == opcode); - } - CHECK(opcode != INSTRUMENTED_LINE); - } - else if (data->lines) { - CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == 255); - } - if (is_instrumented(opcode)) { - int deinstrumented = DE_INSTRUMENT[opcode]; - CHECK(valid_opcode(deinstrumented)); - CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - if (_PyOpcode_Caches[deinstrumented]) { - CHECK(instr[1].cache > 0); - } - opcode = deinstrumented; - int event = EVENT_FOR_OPCODE[opcode]; - if (event < 0) { - /* RESUME fixup */ - event = instr->oparg; - } - CHECK(tools_matrix.tools[event] != 0); - } - if (data->lines && opcode != END_FOR) { - int line1 = compute_line(code, i, data->lines[i].line_delta); - int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); - CHECK(line1 == line2); - } - CHECK(valid_opcode(opcode)); - opcode = _PyOpcode_Deopt[opcode]; - CHECK(valid_opcode(opcode)); - if (data->tools) { - uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; - if (event == -1) { - /* RESUME fixup */ - event = instr->oparg; - } - uint8_t global_tools = tools_matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); - } - else { - CHECK(local_tools == 0xff); - } - } - i += _PyOpcode_Caches[opcode]; - } -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; } - /* To do -- Handle INSTRUMENTED_INSTRUCTION if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcode[offset]; + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; } - */ if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; - deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } } - assert(!is_instrumented(opcode)); - assert(_PyOpcode_Deopt[opcode]); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + assert(_PyOpcode_Deopt[opcode] != 0); return _PyOpcode_Deopt[opcode]; } static void de_instrument(PyCodeObject *code, int offset, int event) { + assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (!is_instrumented(opcode)) { return; } - int deinstrumented = DE_INSTRUMENT[opcode]; - int base_opcode; - if (deinstrumented) { - base_opcode = deinstrumented; - instr->opcode = base_opcode; - } - else { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - assert(opcode == INSTRUMENTED_LINE); - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; - int orignal_opcode = lines->original_opcode; - if (is_instrumented(orignal_opcode)) { - base_opcode = DE_INSTRUMENT[orignal_opcode]; - lines->original_opcode = DE_INSTRUMENT[orignal_opcode]; - assert(lines->original_opcode != 0); - + switch(opcode) { + case INSTRUMENTED_INSTRUCTION: + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + if (opcode != INSTRUMENTED_LINE) { + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = deinstrumented; + } + break; + } + /* Intentional fall through */ + case INSTRUMENTED_LINE: + { + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int orignal_opcode = lines->original_opcode; + int deinstrumented = DE_INSTRUMENT[orignal_opcode]; + if (deinstrumented) { + lines->original_opcode = deinstrumented; + } + break; } - else { - base_opcode = orignal_opcode; + default: + { + int deinstrumented = DE_INSTRUMENT[opcode]; + assert(deinstrumented); + instr->opcode = deinstrumented; } } + int base_opcode = _Py_GetBaseOpcode(code, offset); assert(base_opcode != 0); assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { @@ -500,21 +562,58 @@ de_instrument(PyCodeObject *code, int offset, int event) } static void -de_instrument_line(PyCodeObject *code, int offset) +de_instrument_line(PyCodeObject *code, int i) { /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i]; + } if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; int original_opcode = lines->original_opcode; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->opcode = original_opcode; } + else { + original_opcode = _PyOpcode_Deopt[original_opcode]; + assert(original_opcode != 0); + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); + } + } + if (instr->opcode == INSTRUMENTED_INSTRUCTION) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = original_opcode; + } + else { + instr->opcode = original_opcode; + } + assert(instr->opcode != INSTRUMENTED_LINE); + /* Mark instruction as candidate for line instrumentation */ + lines->original_opcode = 255; +} + + +static void +de_instrument_per_instruction(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + if (opcode != INSTRUMENTED_INSTRUCTION) { + return; + } + int original_opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + if (is_instrumented(original_opcode)) { + /* Instrumented original */ + instr->opcode = original_opcode; + } else { int base_opcode = _PyOpcode_Deopt[original_opcode]; assert(base_opcode != 0); @@ -523,10 +622,12 @@ de_instrument_line(PyCodeObject *code, int offset) instr[1].cache = adaptive_counter_warmup(); } } - /* Mark instruction as candidate for line instrumentation */ - lines->original_opcode = 255; + assert(instr->opcode != INSTRUMENTED_INSTRUCTION); + /* Keep things clean for snaity check */ + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = 0; } + static void instrument(PyCodeObject *code, int offset) { @@ -552,19 +653,16 @@ instrument(PyCodeObject *code, int offset) } static void -instrument_line(PyCodeObject *code, int offset) +instrument_line(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; - if (lines->original_opcode != 255) { - dump_instrumentation_data(code, offset, stderr); - } - assert(lines->original_opcode == 255); + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + CHECK(lines->original_opcode == 255); if (is_instrumented(opcode)) { lines->original_opcode = opcode; } @@ -577,7 +675,30 @@ instrument_line(PyCodeObject *code, int offset) } assert(lines->original_opcode > 0); instr->opcode = INSTRUMENTED_LINE; - assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); + assert(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); +} + +static void +instrument_per_instruction(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + if (opcode == INSTRUMENTED_INSTRUCTION) { + return; + } + CHECK(opcode != 0); + if (is_instrumented(opcode)) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = opcode; + } + else { + assert(opcode != 0); + assert(_PyOpcode_Deopt[opcode] != 0); + assert(_PyOpcode_Deopt[opcode] != RESUME); + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + } + assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] > 0); + instr->opcode = INSTRUMENTED_INSTRUCTION; } #ifndef NDEBUG @@ -586,7 +707,9 @@ instruction_has_event(PyCodeObject *code, int offset) { _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.opcode; - /* TO DO -- Handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; } @@ -678,6 +801,47 @@ add_line_tools(PyCodeObject * code, int offset, int tools) instrument_line(code, offset); } + +static void +add_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools + ) { + code->_co_instrumentation.monitoring_data->per_instruction_tools[offset] |= tools; + } + else { + /* Single tool */ + assert(_Py_popcount32(tools) == 1); + } + instrument_per_instruction(code, offset); +} + + +static void +remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools) + { + uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->per_instruction_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_per_instruction(code, offset); + } + } + else { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_per_instruction(code, offset); + } + } +} + + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( @@ -751,8 +915,6 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); - /* No per-instruction monitoring yet */ - assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -895,11 +1057,6 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); - if (tstate->trace_info.code == frame->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - return original_opcode; - } uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[offset] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; @@ -929,14 +1086,62 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, remove_line_tools(code, offset, 1 << tool); } } - tstate->trace_info.code = frame->f_code; - tstate->trace_info.line = line; Py_DECREF(line_obj); assert(original_opcode != 0); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } +int +_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +{ + PyCodeObject *code = frame->f_code; + assert(is_instrumentation_up_to_date(code, tstate->interp)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); + _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; + assert(instrumentation_data->per_instruction_opcodes); + int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; + if (tstate->tracing) { + return next_opcode; + } + PyInterpreterState *interp = tstate->interp; + uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? + instrumentation_data->per_instruction_tools[offset] : + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, &args[1], + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_INSTRUCTION); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(offset_obj); + return -1; + } + else { + /* DISABLE */ + remove_line_tools(code, offset, 1 << tool); + } + } + Py_DECREF(offset_obj); + assert(next_opcode != 0); + return next_opcode; +} + + PyObject * _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { @@ -997,14 +1202,10 @@ initialize_tools(PyCodeObject *code) } } -static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) -{ - int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); - return line >= 0 && line != current_line; -} - #define LINE_MARKER 255 +#define NO_LINE -128 + static void initialize_lines(PyCodeObject *code) { @@ -1021,58 +1222,69 @@ initialize_lines(PyCodeObject *code) int extended_arg = 0; _Py_CODEUNIT *bytecode = _PyCode_CODE(code); for (int i = code->_co_firsttraceable; i < code_len; i++) { - line_data[i].original_opcode = 0; + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } + line_data[i].line_delta = compute_line_delta(code, i, line); + current_line = line; + int opcode = _Py_GetBaseOpcode(code, i); + for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + } } for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); switch (opcode) { + case END_ASYNC_FOR: case END_FOR: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; - line_data[i].line_delta = -127; + line_data[i].line_delta = NO_LINE; extended_arg = 0; continue; - case EXTENDED_ARG: - extended_arg = (extended_arg << 8) + bytecode[i].oparg; - line_data[i].line_delta = -127; - continue; - case JUMP_IF_FALSE_OR_POP: - case JUMP_IF_TRUE_OR_POP: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - case JUMP_FORWARD: - /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ - { - int offset = (extended_arg << 8) + bytecode[i].oparg; - line_data[i+1+offset].original_opcode = LINE_MARKER; - break; - } - case FOR_ITER: - { - /* Skip over END_FOR */ - int offset = (extended_arg << 8) + bytecode[i].oparg + 1; - line_data[i+1+offset].original_opcode = LINE_MARKER; - break; - } - case JUMP_BACKWARD: - { - int offset = (extended_arg << 8) + bytecode[i].oparg; - line_data[i+1-offset].original_opcode = LINE_MARKER; - break; - } - } - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = LINE_MARKER; - current_line = range.ar_line; - } - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); - for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = -128; +// case EXTENDED_ARG: +// extended_arg = (extended_arg << 8) + bytecode[i].oparg; +// continue; +// case JUMP_IF_FALSE_OR_POP: +// case JUMP_IF_TRUE_OR_POP: +// case JUMP_FORWARD: +// /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ +// case POP_JUMP_IF_FALSE: +// case POP_JUMP_IF_TRUE: +// { +// int target = i + 1 + (extended_arg << 8) + bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } +// case FOR_ITER: +// { +// /* Skip over END_FOR */ +// int target = i + 2 + (extended_arg << 8) + bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } +// case JUMP_BACKWARD: +// { +// int target = i + 1 - (extended_arg << 8) - bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } } + i+= _PyOpcode_Caches[opcode]; extended_arg = 0; } } @@ -1132,6 +1344,30 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } + if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = 0; + } + } + if (multitools && code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + code->_co_instrumentation.monitoring_data->per_instruction_tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data->per_instruction_tools[i] = 0; + } + } + } return 0; } @@ -1159,7 +1395,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); if (update_instrumentation_data(code, interp)) { return -1; @@ -1187,20 +1422,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); - /* TO DO handle INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; - } - int base_opcode; - if (DE_INSTRUMENT[opcode]) { - base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - } - else { - base_opcode = _PyOpcode_Deopt[opcode]; - } if (is_super_instruction(opcode)) { - instr->opcode = base_opcode; + instr->opcode = _PyOpcode_Deopt[opcode]; } + int base_opcode = _Py_GetBaseOpcode(code, i); if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; if (base_opcode == RESUME) { @@ -1221,18 +1446,18 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[base_opcode]; if (base_opcode == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ - assert(instr[2].opcode == POP_JUMP_IF_FALSE || + CHECK(instr[2].opcode == POP_JUMP_IF_FALSE || instr[2].opcode == POP_JUMP_IF_TRUE); i++; } + i += _PyOpcode_Caches[base_opcode]; } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools || removed_line_tools) { + if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; for (int i = code->_co_firsttraceable; i < code_len; i++) { if (line_data[i].original_opcode) { @@ -1252,6 +1477,28 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } + uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + if (new_per_instruction_tools | removed_per_instruction_tools) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == RESUME || opcode == END_FOR) { + continue; + } + if (removed_per_instruction_tools) { + remove_per_instruction_tools(code, i, removed_per_instruction_tools); + } + if (new_per_instruction_tools) { + add_per_instruction_tools(code, i, new_per_instruction_tools); + } + /* TO DO -- Use instruction length, not cache count */ + i += _PyOpcode_Caches[opcode]; + if (opcode == COMPARE_AND_BRANCH) { + /* Skip over following POP_JUMP_IF */ + i++; + } + } + } //sanity_check_instrumentation(code); return 0; } @@ -1293,7 +1540,6 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); } interp->monitoring_version++; - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); instrument_all_executing_code_objects(interp); } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index d232917273f8d0..9d78a910096179 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -146,14 +146,29 @@ sys_trace_func3( } static PyObject * -sys_trace_none_func( +sys_trace_instruction_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - return call_trace_func(self, Py_None); + assert(nargs == 2); + PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_opcodes) { + Py_RETURN_NONE; + } + Py_INCREF(frame); + PyThreadState *tstate = _PyThreadState_GET(); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; } static PyObject * @@ -163,6 +178,8 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } + tstate->trace_info.code = iframe->f_code; + tstate->trace_info.line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -188,33 +205,14 @@ sys_trace_line_func( assert(nargs == 2); _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); - if (frame == NULL) { - return NULL; - } - if (!frame->f_trace_lines) { - Py_RETURN_NONE; - } assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - return trace_line(tstate, iframe, self, frame, line); -} - -static PyObject * -sys_trace_handled_exception( - _PyLegacyEventHandler *self, PyObject *const *args, - size_t nargsf, PyObject *kwnames -) { - assert(kwnames == NULL); - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); if (frame == NULL) { return NULL; @@ -222,21 +220,10 @@ sys_trace_handled_exception( if (!frame->f_trace_lines) { Py_RETURN_NONE; } - assert(args[0] == (PyObject *)iframe->f_code); - int offset = _PyLong_AsInt(args[1]); - assert(offset >= 0); - if (_PyCode_CODE(iframe->f_code)[offset].opcode == END_ASYNC_FOR) { - Py_RETURN_NONE; - } - int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); - if (line < 0) { - Py_RETURN_NONE; - } return trace_line(tstate, iframe, self, frame, line); } - static PyObject * sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -254,10 +241,9 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); /* Forward jumps should have been handled by the line instrumentation. - * We need to handle backward jumps, as the semantics differ. * PEP 669 only generates events when a new line is seen. * Legacy tracing generates events for all backward jumps even on - * the same line */ + * the same line. We handle that case here. */ if (to > from) { /* Forwards jump */ Py_RETURN_NONE; @@ -456,7 +442,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, + (vectorcallfunc)sys_trace_instruction_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } @@ -469,8 +455,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); - /* TO DO -- opcode events */ - if (tstate->interp->f_opcode_trace_set && false) { + if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index daef17b1569ec1..21f69220f20d10 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -185,6 +185,7 @@ static const struct { [BINARY_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [SWAP] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_LINE] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_INSTRUCTION] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [INSTRUMENTED_JUMP_FORWARD] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_BACKWARD] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2750da3f642179..0c4ea0ec2b4a3d 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -237,7 +237,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, From f434ec7669e6d973eba93547a2a4e8922a951a74 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 15:45:48 +0000 Subject: [PATCH 017/116] Get sys.settrace tests passing. --- Lib/test/test_monitoring.py | 2 +- Lib/test/test_sys_settrace.py | 1 + Objects/frameobject.c | 14 +- Python/bytecodes.c | 62 +++---- Python/ceval.c | 30 ++++ Python/generated_cases.c.h | 68 +++---- Python/instrumentation.c | 330 ++++++++++++++++------------------ Python/legacy_tracing.c | 65 +++---- Python/opcode_metadata.h | 2 +- 9 files changed, 273 insertions(+), 301 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1e5ef85eb4dcab..fabf3ab8ae6d05 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 21, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 35b1a0b629d406..9079f0c0ceb4c8 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1705,6 +1705,7 @@ def f(): def g(frame, event, arg): if (event == 'exception'): type, exception, trace = arg + print(arg) self.assertIsInstance(exception, Exception) return g diff --git a/Objects/frameobject.c b/Objects/frameobject.c index dc196846d57ab4..0c15ffb2f8740b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -693,15 +693,17 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } switch(what_event) { - case PY_MONITORING_EVENT_PY_START: - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); - return -1; case PY_MONITORING_EVENT_PY_RESUME: - break; + case PY_MONITORING_EVENT_JUMP: + case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: + /* OK */ break; + case PY_MONITORING_EVENT_PY_START: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; case PY_MONITORING_EVENT_CALL: case PY_MONITORING_EVENT_C_RETURN: PyErr_SetString(PyExc_ValueError, @@ -712,8 +714,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore case PY_MONITORING_EVENT_PY_THROW: case PY_MONITORING_EVENT_RAISE: case PY_MONITORING_EVENT_C_RAISE: - case PY_MONITORING_EVENT_JUMP: - case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_INSTRUCTION: case PY_MONITORING_EVENT_EXCEPTION_HANDLED: PyErr_Format(PyExc_ValueError, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4b3b50bee3f4c4..17dca31131b5a4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -776,13 +776,6 @@ dummy_func( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); ERROR_IF(err, error); - if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -1891,8 +1884,10 @@ dummy_func( } } - inst(INSTRUMENTED_COMPARE_AND_BRANCH, (unused/2, left, right -- )) { + inst(INSTRUMENTED_COMPARE_AND_BRANCH, ( -- )) { assert((oparg >> 4) <= Py_GE); + PyObject *right = POP(); + PyObject *left = POP(); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); Py_DECREF(right); @@ -1900,21 +1895,15 @@ dummy_func( assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + int offset = 0; int err = PyObject_IsTrue(cond); + _Py_CODEUNIT *here = next_instr-1; Py_DECREF(cond); ERROR_IF(err < 0, error); - _Py_CODEUNIT *target; if (jump_on_true == (err != 0)) { - target = next_instr + offset + 2; - JUMPBY(offset); - } - else { - target = next_instr + 2; + offset = _Py_OPARG(next_instr[1]); } - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); - ERROR_IF(err, error); + INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); } inst(COMPARE_AND_BRANCH_FLOAT, (unused/2, left, right -- )) { @@ -3426,19 +3415,19 @@ dummy_func( } inst(INSTRUMENTED_LINE, ( -- )) { + _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame, next_instr-1); - ERROR_IF(original_opcode < 0, error); - next_instr--; - if (frame->prev_instr != next_instr) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { DISPATCH(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3464,19 +3453,12 @@ dummy_func( // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); - ERROR_IF(err, error); - JUMPBY(oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); } // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { - assert(oparg < INSTR_OFFSET()); - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr-oparg); - ERROR_IF(err, error); - JUMPBY(-oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); } @@ -3487,8 +3469,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); if (err) { JUMPBY(oparg); @@ -3506,8 +3490,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); if (err == 0) { JUMPBY(oparg); @@ -3525,8 +3511,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); JUMPBY(offset); } @@ -3538,8 +3526,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); JUMPBY(offset); } diff --git a/Python/ceval.c b/Python/ceval.c index 1e2a08555e3f3a..8716ab82dad5bb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,6 +217,19 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" +#define UPDATE_NEXT_OR_JUMP(offset) \ +do { \ + assert(frame->stacktop >= 0); \ + if (frame->prev_instr != next_instr-1) { \ + next_instr = frame->prev_instr; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + } \ + else { \ + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); \ + JUMPBY(offset); \ + } \ + frame->stacktop = -1; \ +} while (0) #ifdef HAVE_ERRNO_H #include @@ -913,6 +926,22 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + frame->stacktop = -1; \ + if (err) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ + next_instr = frame->prev_instr; \ +} while (0); + // GH-89279: Must be a macro to be sure it's inlined by MSVC. #define is_method(stack_pointer, args) (PEEK((args)+2) != NULL) @@ -2234,6 +2263,7 @@ monitor_raise(PyThreadState *tstate, } _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; + assert(value != NULL && value != Py_None); arg = PyTuple_Pack(3, type, value, traceback); if (arg == NULL) { _PyErr_Restore(tstate, type, value, orig_traceback); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 317485009c8193..fd894d477c9294 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -941,13 +941,6 @@ tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); if (err) goto error; - if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -2151,33 +2144,25 @@ } TARGET(INSTRUMENTED_COMPARE_AND_BRANCH) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); assert((oparg >> 4) <= Py_GE); + PyObject *right = POP(); + PyObject *left = POP(); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); Py_DECREF(right); - if (cond == NULL) goto pop_2_error; + if (cond == NULL) goto error; assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + int offset = 0; int err = PyObject_IsTrue(cond); + _Py_CODEUNIT *here = next_instr-1; Py_DECREF(cond); - if (err < 0) goto pop_2_error; - _Py_CODEUNIT *target; + if (err < 0) goto error; if (jump_on_true == (err != 0)) { - target = next_instr + offset + 2; - JUMPBY(offset); - } - else { - target = next_instr + 2; + offset = _Py_OPARG(next_instr[1]); } - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); - if (err) goto pop_2_error; - STACK_SHRINK(2); - JUMPBY(2); + INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3753,19 +3738,19 @@ } TARGET(INSTRUMENTED_LINE) { + _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame, next_instr-1); - if (original_opcode < 0) goto error; - next_instr--; - if (frame->prev_instr != next_instr) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { DISPATCH(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3790,19 +3775,12 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); - if (err) goto error; - JUMPBY(oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(oparg < INSTR_OFFSET()); - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr-oparg); - if (err) goto error; - JUMPBY(-oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3814,8 +3792,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; if (err) { JUMPBY(oparg); @@ -3834,8 +3814,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; if (err == 0) { JUMPBY(oparg); @@ -3854,8 +3836,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3868,8 +3852,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; JUMPBY(offset); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0ade817707bc48..4e8ed0d00ac2af 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,6 +7,7 @@ #include "pycore_opcode.h" #include "pycore_pyerrors.h" +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -126,6 +127,133 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +static inline bool +is_instrumented(int opcode) { + assert(opcode != 0); + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } + } + return true; +} + +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(val == 0 || val == 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + +/* Line delta. + * 8 bit value. + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + */ + +#define OFFSET_SHIFT 4 + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + if (line < 0) { + return -128; + } + // assert(line >= code->co_firstlineno); + // assert(offset >= code->_co_firsttraceable); + // assert(offset < Py_SIZE(code)); + int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); + if (delta < 128 && delta > -128) { + return delta; + } + return -127; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta > -127) { + // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); + return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } +} + + #ifdef INSTRUMENT_DEBUG static void @@ -367,133 +495,6 @@ sanity_check_instrumentation(PyCodeObject *code) #endif -#define OFFSET_SHIFT 4 - -/* Line delta. - * 8 bit value. - * if line_delta == -128: - * line = None # represented as -1 - * elif line == -127: - * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - * else: - * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; - */ - -static int8_t -compute_line_delta(PyCodeObject *code, int offset, int line) -{ - if (line < 0) { - return -128; - } - // assert(line >= code->co_firstlineno); - // assert(offset >= code->_co_firsttraceable); - // assert(offset < Py_SIZE(code)); - int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); - if (delta < 128 && delta > -128) { - return delta; - } - return -127; -} - -static int -compute_line(PyCodeObject *code, int offset, int8_t line_delta) -{ - if (line_delta > -127) { - // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); - return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; - } - if (line_delta == -128) { - return -1; - } - else { - assert(line_delta == -127); - /* Compute from table */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } -} - - -static inline bool -is_instrumented(int opcode) { - assert(opcode != 0); - return INSTRUMENTED_OPCODES[opcode] == opcode; -} - -static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (a.tools[i] != b.tools[i]) { - return false; - } - } - return true; -} - -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & ~b.tools[i]; - } - return res; -} - -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & b.tools[i]; - } - return res; -} - -static inline bool -matrix_empty(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (m.tools[i]) { - return false; - } - } - return true; -} - -static inline int -multiple_tools(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { - return true; - } - } - return false; -} - -static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); -} - -static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) -{ - _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - return result; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -675,7 +676,7 @@ instrument_line(PyCodeObject *code, int i) } assert(lines->original_opcode > 0); instr->opcode = INSTRUMENTED_LINE; - assert(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); + CHECK(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); } static void @@ -980,6 +981,7 @@ _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target ) { + frame->prev_instr = target; PyCodeObject *code = frame->f_code; int from = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); @@ -1044,6 +1046,7 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { + frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); @@ -1219,9 +1222,18 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; - int extended_arg = 0; - _Py_CODEUNIT *bytecode = _PyCode_CODE(code); for (int i = code->_co_firsttraceable; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + switch (opcode) { + case END_ASYNC_FOR: + case END_FOR: + case RESUME: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + continue; + } int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); if (line != current_line && line >= 0) { line_data[i].original_opcode = LINE_MARKER; @@ -1230,62 +1242,22 @@ initialize_lines(PyCodeObject *code) line_data[i].original_opcode = 0; } line_data[i].line_delta = compute_line_delta(code, i, line); - current_line = line; - int opcode = _Py_GetBaseOpcode(code, i); + if (line >= 0) { + current_line = line; + } for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; line_data[i].line_delta = NO_LINE; } - } - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); switch (opcode) { - case END_ASYNC_FOR: - case END_FOR: - case RESUME: - /* END_FOR cannot start a line, as it is skipped by FOR_ITER - * RESUME must not be instrumented with INSTRUMENT_LINE */ - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - extended_arg = 0; - continue; -// case EXTENDED_ARG: -// extended_arg = (extended_arg << 8) + bytecode[i].oparg; -// continue; -// case JUMP_IF_FALSE_OR_POP: -// case JUMP_IF_TRUE_OR_POP: -// case JUMP_FORWARD: -// /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ -// case POP_JUMP_IF_FALSE: -// case POP_JUMP_IF_TRUE: -// { -// int target = i + 1 + (extended_arg << 8) + bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } -// case FOR_ITER: -// { -// /* Skip over END_FOR */ -// int target = i + 2 + (extended_arg << 8) + bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } -// case JUMP_BACKWARD: -// { -// int target = i + 1 - (extended_arg << 8) - bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } + case RETURN_VALUE: + case RAISE_VARARGS: + case RERAISE: + /* Blocks of code after these terminators + * should be treated as different lines */ + current_line = -1; } - i+= _PyOpcode_Caches[opcode]; - extended_arg = 0; } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 9d78a910096179..989ce7852b5edb 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -56,8 +56,7 @@ sys_profile_func2( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); return call_profile_func(self, Py_None); } @@ -67,8 +66,7 @@ sys_profile_func3( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); return call_profile_func(self, args[2]); } @@ -78,8 +76,7 @@ sys_profile_call_or_return( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); PyObject *callable = args[2]; if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { Py_RETURN_NONE; @@ -111,15 +108,10 @@ sys_trace_exception_func( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - _PyErr_StackItem *err_info = _PyErr_GetTopmostException(_PyThreadState_GET()); - PyObject *arg = _PyErr_StackItemToExcInfoTuple(err_info); - if (arg == NULL) { - return NULL; - } + assert(PyVectorcall_NARGS(nargsf) == 3); + PyObject *arg = args[2]; + assert(PyTuple_CheckExact(arg)); PyObject *res = call_trace_func(self, arg); - Py_DECREF(arg); return res; } @@ -129,8 +121,7 @@ sys_trace_func2( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); return call_trace_func(self, Py_None); } @@ -140,8 +131,7 @@ sys_trace_func3( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); return call_trace_func(self, args[2]); } @@ -151,8 +141,7 @@ sys_trace_instruction_func( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; @@ -201,8 +190,7 @@ sys_trace_line_func( if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); assert(args[0] == (PyObject *)iframe->f_code); @@ -234,20 +222,11 @@ sys_trace_jump_func( if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); int from = _PyLong_AsInt(args[1]); assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - /* Forward jumps should have been handled by the line instrumentation. - * PEP 669 only generates events when a new line is seen. - * Legacy tracing generates events for all backward jumps even on - * the same line. We handle that case here. */ - if (to > from) { - /* Forwards jump */ - Py_RETURN_NONE; - } _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); @@ -258,8 +237,22 @@ sys_trace_jump_func( Py_RETURN_NONE; } /* We can call _Py_Instrumentation_GetLine because we always set - * line events for tracing */ + * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + /* Backward jump: Always generate event + * Forward jump: Only generate event if jumping to different line. */ + if (to > from) { + /* Forwards jump */ + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == to_line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (from_line == to_line) { + Py_RETURN_NONE; + } + } return trace_line(tstate, iframe, self, frame, to_line); } @@ -438,7 +431,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, -1)) { + PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -453,8 +446,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 21f69220f20d10..9ad33884808bf7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -117,7 +117,7 @@ static const struct { [STORE_ATTR_SLOT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, [COMPARE_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [COMPARE_AND_BRANCH] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [INSTRUMENTED_COMPARE_AND_BRANCH] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [INSTRUMENTED_COMPARE_AND_BRANCH] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [COMPARE_AND_BRANCH_FLOAT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_INT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_STR] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, From b680084a913234528c3586c624005e3a39c97fbf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 16:59:29 +0000 Subject: [PATCH 018/116] Monitor 'internal' StopIteration raises. --- Python/bytecodes.c | 8 +++----- Python/generated_cases.c.h | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 17dca31131b5a4..a935769db19f29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -737,9 +737,9 @@ dummy_func( retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -2311,9 +2311,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - monitor_raise(tstate, frame, next_instr-1); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fd894d477c9294..e08c74a1b7124d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -901,9 +901,9 @@ retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -2619,9 +2619,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - monitor_raise(tstate, frame, next_instr-1); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ From e6e7cf1f21567cbe2bb506b83a05913a83589d8f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 17:14:02 +0000 Subject: [PATCH 019/116] Check for NULLs. --- Python/instrumentation.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4e8ed0d00ac2af..350e35bd25da6a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -954,6 +954,9 @@ _Py_call_instrumentation( assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + if (instruction_offset_obj == NULL) { + return -1; + } PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); @@ -970,6 +973,9 @@ _Py_call_instrumentation_arg( assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + if (instruction_offset_obj == NULL) { + return -1; + } PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); From 5bbc83e4d45c32f3f525ea9e44d40539cf620b37 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 17:30:41 +0000 Subject: [PATCH 020/116] Fix up a few tests --- Lib/test/test__opcode.py | 4 ++-- Lib/test/test_code.py | 8 ++++---- Lib/test/test_dis.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 4e5860dfbe6ca3..5df05b6f57ded6 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,7 +20,7 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 240: + if code >= 238: # Opcodes 240 and up are internal instrumentation instructions continue with self.subTest(opname=name): @@ -54,7 +54,7 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 240: + if code >= 238: # Opcodes 240 and up are internal instrumentation instructions continue with self.subTest(opname=name): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 67ed1694205cd6..bf5a6d39281e04 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -343,13 +343,13 @@ def foo(): pass # assert that opcode 238 is invalid - self.assertEqual(opname[238], '<238>') + self.assertEqual(opname[235], '<235>') - # change first opcode to 0xee (=238) + # change first opcode to 0xeb (=235) foo.__code__ = foo.__code__.replace( - co_code=b'\xee' + foo.__code__.co_code[1:]) + co_code=b'\xeb' + foo.__code__.co_code[1:]) - msg = f"unknown opcode 238" + msg = f"unknown opcode 235" with self.assertRaisesRegex(SystemError, msg): foo() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 894c412cac7704..bc5ce2bfe9f976 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -895,7 +895,7 @@ def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'INSTRUMENTED_CALL_FUNCTION_EX']) for opcode, opname in enumerate(dis.opname): - if opname in long_opcodes: + if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH From 7fe9a4354e69be761a9bb0124275389daa53dddb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 23 Jan 2023 10:42:44 +0000 Subject: [PATCH 021/116] Turn off debugging output by default. --- Python/instrumentation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 350e35bd25da6a..5ca720f2e4301e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,7 +7,8 @@ #include "pycore_opcode.h" #include "pycore_pyerrors.h" -#define INSTRUMENT_DEBUG 1 +/* Uncomment this to dump debugging output when assertions fail */ +//#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { From 8b9f996afbf7df561b3b32c78ce22bd85a6878fb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 13:56:28 +0000 Subject: [PATCH 022/116] Remove debugging printfs --- Objects/frameobject.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0c15ffb2f8740b..30a083da09025a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -839,7 +839,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { - fprintf(stderr, "Popping exception\n"); /* Pop exception stack as well as the evaluation stack */ PyThreadState *tstate = _PyThreadState_GET(); _PyErr_StackItem *exc_info = tstate->exc_info; @@ -850,13 +849,11 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore Py_XDECREF(value); } else { - fprintf(stderr, "Popping object\n"); PyObject *v = _PyFrame_StackPop(f->f_frame); Py_XDECREF(v); } start_stack = pop_value(start_stack); } - fprintf(stderr, "Jumping to %d\n", best_addr); /* Finally set the new lasti and return OK. */ f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; From 9b026404be3ab31e0fe3ea6468a94e3e64c7019c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 14:38:31 +0000 Subject: [PATCH 023/116] Avoid refleak. --- Python/legacy_tracing.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 989ce7852b5edb..f8b6a4d98fe2a9 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -144,6 +144,7 @@ sys_trace_instruction_func( assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + /* No frame */ return NULL; } if (!frame->f_trace_opcodes) { @@ -387,15 +388,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - int delta = (func != NULL) - (tstate->c_tracefunc != NULL); - tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; - tstate->c_traceobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - /* Setup PEP 669 monitoring callbacks and events. */ - tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; @@ -441,6 +434,14 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } + + tstate->c_tracefunc = func; + PyObject *old_traceobj = tstate->c_traceobj; + tstate->c_traceobj = Py_XNewRef(arg); + Py_XDECREF(old_traceobj); + + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); + tstate->interp->sys_tracing_threads += delta; if (tstate->interp->sys_tracing_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | @@ -456,6 +457,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) else { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); } - Py_XDECREF(old_traceobj); + return 0; } From 2cadf32e57228d4a8e84af8bf415630fa5f52ac0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 16:56:13 +0000 Subject: [PATCH 024/116] Record last traced line on frame object. --- Include/cpython/pystate.h | 8 ------ Include/internal/pycore_frame.h | 1 + Objects/frameobject.c | 1 + Python/legacy_tracing.c | 44 ++++++++++++++------------------- 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0c70cbb1b05183..0a0f117b782b8d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -55,12 +55,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 - -typedef struct { - PyCodeObject *code; // The code object for the line number. May be NULL. - int line; // Only valid if code != NULL. -} PyTraceInfo; - // Internal structure: you should not use it directly, but use public functions // like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). typedef struct _PyCFrame { @@ -204,8 +198,6 @@ struct _ts { /* Unique thread state id. */ uint64_t id; - PyTraceInfo trace_info; - _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f12b225ebfccf2..76ef105cb4c475 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,6 +19,7 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ + int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 30a083da09025a..6104549f0c8454 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1076,6 +1076,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_last_traced_line = -1; return f; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index f8b6a4d98fe2a9..6ca35d8171ac48 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -168,8 +168,7 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } - tstate->trace_info.code = iframe->f_code; - tstate->trace_info.line = line; + frame ->f_last_traced_line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -197,16 +196,12 @@ sys_trace_line_func( assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); if (frame == NULL) { return NULL; } - if (!frame->f_trace_lines) { + if (frame ->f_last_traced_line == line || !frame->f_trace_lines) { + /* Already traced this line */ Py_RETURN_NONE; } return trace_line(tstate, iframe, self, frame, line); @@ -244,8 +239,7 @@ sys_trace_jump_func( * Forward jump: Only generate event if jumping to different line. */ if (to > from) { /* Forwards jump */ - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == to_line) { + if (frame->f_last_traced_line == to_line) { /* Already traced this line */ Py_RETURN_NONE; } @@ -314,16 +308,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { return -1; } - int delta = (func != NULL) - (tstate->c_profilefunc != NULL); - tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; - tstate->c_profileobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - /* Setup PEP 669 monitoring callbacks and events. */ - tstate->interp->sys_profiling_threads += delta; - assert(tstate->interp->sys_profiling_threads >= 0); if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, @@ -357,7 +342,16 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } } - if (tstate->interp->sys_profiling_threads && delta) { + + int delta = (func != NULL) - (tstate->c_profilefunc != NULL); + tstate->c_profilefunc = func; + PyObject *old_profileobj = tstate->c_profileobj; + tstate->c_profileobj = Py_XNewRef(arg); + Py_XDECREF(old_profileobj); + tstate->interp->sys_profiling_threads += delta; + assert(tstate->interp->sys_profiling_threads >= 0); + + if (tstate->interp->sys_profiling_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | @@ -368,9 +362,6 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) else if (tstate->interp->sys_profiling_threads == 0) { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); } - // gh-98257: Only call Py_XDECREF() once the new profile function is fully - // set, so it's safe to call sys.setprofile() again (reentrant call). - Py_XDECREF(old_profileobj); return 0; } @@ -388,8 +379,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - /* Setup PEP 669 monitoring callbacks and events. */ assert(tstate->interp->sys_tracing_threads >= 0); + /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -435,13 +426,14 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; PyObject *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); Py_XDECREF(old_traceobj); - - int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->interp->sys_tracing_threads += delta; + assert(tstate->interp->sys_tracing_threads >= 0); + if (tstate->interp->sys_tracing_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | From 1f54d7721aab4b968db9607cbfa4ba24a27d9db5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 10:09:10 +0000 Subject: [PATCH 025/116] Get a couple more top-level tests passing. --- Lib/test/test_sys.py | 2 +- Objects/codeobject.c | 18 ++++++++ Python/bytecodes.c | 1 - Python/generated_cases.c.h | 1 - Python/instrumentation.c | 87 +++++++++++++++++++++----------------- Python/legacy_tracing.c | 33 +++++++-------- 6 files changed, 84 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ab1a0659471857..52f4cc886dfd80 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1443,7 +1443,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pii3c7P2ic??2P')) # function def func(): pass check(func, size('14Pi')) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 535a521e3170b0..1d23689bdcac7c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1703,6 +1703,24 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } + _PyCoInstrumentationData *data = co->_co_instrumentation.monitoring_data; + if (data) { + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + } if (co->_co_linearray) { PyMem_Free(co->_co_linearray); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a935769db19f29..a4566f574f393f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3430,7 +3430,6 @@ dummy_func( _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); } - assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; DISPATCH_GOTO(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e08c74a1b7124d..c6fff9076bfc14 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3753,7 +3753,6 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); } - assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; DISPATCH_GOTO(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5ca720f2e4301e..2bc7e9ee6beb51 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -8,7 +8,7 @@ #include "pycore_pyerrors.h" /* Uncomment this to dump debugging output when assertions fail */ -//#define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -889,35 +889,32 @@ static inline int most_significant_bit(uint8_t bits) { } static inline uint8_t -get_tools(PyCodeObject * code, int index, int event) +get_tools_for_instruction(PyCodeObject * code, int i, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (monitoring && monitoring->tools) { - tools = monitoring->tools[index]; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring && monitoring->tools) { + tools = monitoring->tools[i]; } else { tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; } - assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + CHECK((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + CHECK((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); return tools; } static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) + PyObject **args, Py_ssize_t nargsf, int offset, uint8_t tools) { //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } PyInterpreterState *interp = tstate->interp; - int offset = instr - _PyCode_CODE(code); - uint8_t tools = get_tools(code, offset, event); - // assert(tools); while (tools) { int tool = most_significant_bit(tools); assert(tool >= 0 && tool < 8); @@ -953,14 +950,15 @@ _Py_call_instrumentation( PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - if (instruction_offset_obj == NULL) { + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { return -1; } - PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); - Py_DECREF(instruction_offset_obj); + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); return err; } @@ -972,14 +970,15 @@ _Py_call_instrumentation_arg( PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - if (instruction_offset_obj == NULL) { + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { return -1; } - PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); - Py_DECREF(instruction_offset_obj); + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); return err; } @@ -988,21 +987,24 @@ _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target ) { + assert(event == PY_MONITORING_EVENT_JUMP || + event == PY_MONITORING_EVENT_BRANCH); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int from = instr - _PyCode_CODE(code); + int offset = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); PyObject *to_obj = PyLong_FromLong(to); if (to_obj == NULL) { return -1; } - PyObject *from_obj = PyLong_FromLong(from); + PyObject *from_obj = PyLong_FromLong(offset); if (from_obj == NULL) { Py_DECREF(to_obj); return -1; } + uint8_t tools = get_tools_for_instruction(code, offset, event); PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(to_obj); Py_DECREF(from_obj); return err; @@ -1019,11 +1021,19 @@ _Py_call_instrumentation_exc( PyCodeObject *code = frame->f_code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + int err; + if (offset_obj == NULL) { + err = -1; + } + else { + PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; + Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); + uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); + Py_DECREF(offset_obj); + } if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -1033,7 +1043,6 @@ _Py_call_instrumentation_exc( _PyErr_Restore(tstate, type, value, traceback); } assert(_PyErr_Occurred(tstate)); - Py_DECREF(instruction_offset_obj); } int @@ -1057,18 +1066,18 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int offset = instr - _PyCode_CODE(code); + int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; uint8_t original_opcode = line_data->original_opcode; if (tstate->tracing) { - return original_opcode; + goto done; } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; - int line = compute_line(code, offset, line_delta); + int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? - code->_co_instrumentation.monitoring_data->line_tools[offset] : + code->_co_instrumentation.monitoring_data->line_tools[i] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { @@ -1093,11 +1102,13 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ - remove_line_tools(code, offset, 1 << tool); + remove_line_tools(code, i, 1 << tool); } } Py_DECREF(line_obj); +done: assert(original_opcode != 0); + assert(original_opcode < INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } @@ -1143,7 +1154,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ - remove_line_tools(code, offset, 1 << tool); + remove_per_instruction_tools(code, offset, 1 << tool); } } Py_DECREF(offset_obj); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 6ca35d8171ac48..0f176c0b1b0e74 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -162,8 +162,9 @@ sys_trace_instruction_func( } static PyObject * -trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, - _PyLegacyEventHandler *self, PyFrameObject* frame, int line +trace_line( + PyThreadState *tstate, _PyLegacyEventHandler *self, + PyFrameObject* frame, int line ) { if (line < 0) { Py_RETURN_NONE; @@ -191,20 +192,21 @@ sys_trace_line_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 2); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); - assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; } - if (frame ->f_last_traced_line == line || !frame->f_trace_lines) { + assert(args[0] == (PyObject *)frame->f_frame->f_code); + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + if (frame ->f_last_traced_line == line) { /* Already traced this line */ Py_RETURN_NONE; } - return trace_line(tstate, iframe, self, frame, line); + return trace_line(tstate, self, frame, line); } @@ -223,18 +225,19 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; } if (!frame->f_trace_lines) { Py_RETURN_NONE; } + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ - int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + int to_line = _Py_Instrumentation_GetLine(code, to); /* Backward jump: Always generate event * Forward jump: Only generate event if jumping to different line. */ if (to > from) { @@ -243,12 +246,8 @@ sys_trace_jump_func( /* Already traced this line */ Py_RETURN_NONE; } - int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); - if (from_line == to_line) { - Py_RETURN_NONE; - } } - return trace_line(tstate, iframe, self, frame, to_line); + return trace_line(tstate, self, frame, to_line); } From 43a3f3eeed96b1f5612c12d4cfb11efa34f1a109 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:04 +0000 Subject: [PATCH 026/116] Update magic number --- Lib/importlib/_bootstrap_external.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5a8cbb98ed3f56..208e37c657c8fc 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -431,6 +431,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg) # Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction) +# Python 3.12a? 3530 (Add instrumentation support) + # Python 3.13 will start with 3550 # MAGIC must change whenever the bytecode emitted by the compiler may no @@ -442,7 +444,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3530).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c From 3d436cfa91f57b1d0a96bb2c379532bf7958ac1e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:31 +0000 Subject: [PATCH 027/116] Remove debug print statement. --- Lib/profile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/profile.py b/Lib/profile.py index 747ae117dae437..43a7d5ed5f6516 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,7 +187,6 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ - print(event, file = sys.stderr) if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] From 691bcf51b368abf2ba4342e16ed74e572844e835 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:56 +0000 Subject: [PATCH 028/116] Raise SystemError if frame is missing. --- Python/legacy_tracing.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 0f176c0b1b0e74..22af735304df4a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -16,7 +16,6 @@ typedef struct _PyLegacyEventHandler { PyObject_HEAD vectorcallfunc vectorcall; int event; - PyCodeObject *last_code; } _PyLegacyEventHandler; static void @@ -40,8 +39,12 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling profile function."); + return NULL; + } Py_INCREF(frame); - assert(frame != NULL); int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); Py_DECREF(frame); if (err) { @@ -92,8 +95,12 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } Py_INCREF(frame); - assert(frame != NULL); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); Py_DECREF(frame); if (err) { @@ -144,7 +151,8 @@ sys_trace_instruction_func( assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { - /* No frame */ + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } if (!frame->f_trace_opcodes) { @@ -196,6 +204,8 @@ sys_trace_line_func( assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); @@ -227,6 +237,8 @@ sys_trace_jump_func( assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } if (!frame->f_trace_lines) { @@ -254,7 +266,7 @@ sys_trace_jump_func( PyTypeObject _PyLegacyEventHandler_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "sys.profile_event_handler", + "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), .tp_dealloc = (destructor)dealloc, .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), @@ -358,7 +370,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_C_RETURN) | (1 << PY_MONITORING_EVENT_C_RAISE); _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); } - else if (tstate->interp->sys_profiling_threads == 0) { + else { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); } return 0; From f07be070fa4970cf4eaa8883ce08ca24a3f8f782 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 31 Jan 2023 16:36:09 +0000 Subject: [PATCH 029/116] Restore a few tests and better handle EXTENDED_ARG. --- Include/cpython/pystate.h | 1 - Lib/profile.py | 3 +- Lib/test/test_monitoring.py | 2 +- Lib/test/test_sys_settrace.py | 13 ++- Python/instrumentation.c | 191 ++++++++++++++++++++++++---------- Python/pystate.c | 1 - 6 files changed, 145 insertions(+), 66 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0a0f117b782b8d..f8ec16e2b37282 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -124,7 +124,6 @@ struct _ts { the trace/profile. */ int tracing; int what_event; /* The event currently being monitored, if any. */ - uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Lib/profile.py b/Lib/profile.py index 43a7d5ed5f6516..453e56285c510c 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,6 +187,7 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ + if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] @@ -288,7 +289,7 @@ def trace_dispatch_c_call (self, frame, t): def trace_dispatch_return(self, frame, t): if frame is not self.cur[-2]: - assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3], t) + assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3]) self.trace_dispatch_return(self.cur[-2], 0) # Prefix "r" means part of the Returning or exiting frame. diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fabf3ab8ae6d05..1388a745b16992 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 22, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9079f0c0ceb4c8..a251b2272e95eb 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1705,7 +1705,6 @@ def f(): def g(frame, event, arg): if (event == 'exception'): type, exception, trace = arg - print(arg) self.assertIsInstance(exception, Exception) return g @@ -1975,12 +1974,12 @@ def test_no_jump_over_return_try_finally_in_finally_block(output): pass output.append(12) - #@jump_test(3, 4, [1], (ValueError, 'after')) - #def test_no_jump_infinite_while_loop(output): - #output.append(1) - #while True: - #output.append(3) - #output.append(4) + @jump_test(3, 4, [1], (ValueError, 'after')) + def test_no_jump_infinite_while_loop(output): + output.append(1) + while True: + output.append(3) + output.append(4) @jump_test(2, 4, [4, 4]) def test_jump_forwards_into_while_block(output): diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 2bc7e9ee6beb51..e92f1405687bcb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -6,6 +6,7 @@ #include "pycore_namespace.h" #include "pycore_opcode.h" #include "pycore_pyerrors.h" +#include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ #define INSTRUMENT_DEBUG 1 @@ -254,6 +255,97 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } +/* We need to parse instructions from code objects a lot, + * so add a couple of utilities */ +int _Py_GetBaseOpcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; +} + +uint8_t deinstrument(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + return opcode; +} + +typedef struct _Instruction { + uint8_t extended_args; + uint8_t deinstrumented_opcode; + uint8_t actual_opcode; + uint8_t length; + /* Whether a prefix instrument like INSTRUMENTED_INSTRUCTION is attached */ + bool is_prefix_instrumented; + /* Whether a specialized instruction like INSTRUMENTED_RETURN_VALUE is used */ + bool is_specialized_instrumented; +} Instruction; + +Instruction read_instruction(PyCodeObject *code, int offset) +{ + Instruction result = (Instruction){0}; + int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].opcode; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + result.is_prefix_instrumented = true; + } + + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + result.is_prefix_instrumented = true; + } + while (opcode == EXTENDED_ARG) { + result.extended_args += 1; + offset++; + opcode = _PyCode_CODE(code)[offset].opcode; + } + result.length = result.extended_args + 1; + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + result.deinstrumented_opcode = deinstrumented; + result.is_specialized_instrumented = true; + } + else { + result.deinstrumented_opcode = opcode; + } + int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; + if (base == COMPARE_AND_BRANCH) { + /* Skip over following POP_JUMP_IF */ + result.length += 1; + } + else { + result.length += _PyOpcode_Caches[base]; + } + return result; +} #ifdef INSTRUMENT_DEBUG @@ -429,14 +521,14 @@ sanity_check_instrumentation(PyCodeObject *code) tools_matrix) ); int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->opcode; + Instruction inst; + for (int i = 0; i < code_len;) { + inst = read_instruction(code, i); /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { + if (inst.actual_opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); - opcode = data->lines[i].original_opcode; + int opcode = data->lines[i].original_opcode; CHECK(opcode != END_FOR); CHECK(opcode != RESUME); CHECK(opcode != INSTRUMENTED_RESUME); @@ -449,36 +541,32 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(data->lines[i].original_opcode == 0 || data->lines[i].original_opcode == 255); } - if (is_instrumented(opcode)) { - int deinstrumented = DE_INSTRUMENT[opcode]; - CHECK(valid_opcode(deinstrumented)); - CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - if (_PyOpcode_Caches[deinstrumented]) { - CHECK(instr[1].cache > 0); - } - opcode = deinstrumented; - int event = EVENT_FOR_OPCODE[opcode]; + if (inst.is_specialized_instrumented) { + int event = EVENT_FOR_OPCODE[inst.deinstrumented_opcode]; if (event < 0) { /* RESUME fixup */ - event = instr->oparg; + event = _PyCode_CODE(code)[i + inst.extended_args].oparg; } CHECK(tools_matrix.tools[event] != 0); + CHECK(inst.deinstrumented_opcode != END_FOR); } - if (data->lines && opcode != END_FOR) { + if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } - CHECK(valid_opcode(opcode)); - opcode = _PyOpcode_Deopt[opcode]; - CHECK(valid_opcode(opcode)); + CHECK(valid_opcode(inst.deinstrumented_opcode)); + if (inst.is_prefix_instrumented || inst.is_specialized_instrumented) { + CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); + } if (data->tools) { - uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; + uint8_t local_tools = data->tools[i + inst.extended_args]; + int deopt = _PyOpcode_Deopt[inst.deinstrumented_opcode]; + if (OPCODE_HAS_EVENT[deopt]) { + int event = EVENT_FOR_OPCODE[deopt]; if (event == -1) { /* RESUME fixup */ - event = instr->oparg; + event = _PyCode_CODE(code)[i + inst.extended_args].oparg; } uint8_t global_tools = tools_matrix.tools[event]; CHECK((global_tools & local_tools) == local_tools); @@ -487,7 +575,8 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(local_tools == 0xff); } } - i += _PyOpcode_Caches[opcode]; + i += inst.length; + assert(i <= code_len); } } #else @@ -496,27 +585,6 @@ sanity_check_instrumentation(PyCodeObject *code) #endif -int _Py_GetBaseOpcode(PyCodeObject *code, int offset) -{ - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; -} - static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -1242,6 +1310,13 @@ initialize_lines(PyCodeObject *code) int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } switch (opcode) { case END_ASYNC_FOR: case END_FOR: @@ -1249,20 +1324,17 @@ initialize_lines(PyCodeObject *code) /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - continue; - } - int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); - if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; - } - else { - line_data[i].original_opcode = 0; } line_data[i].line_delta = compute_line_delta(code, i, line); if (line >= 0) { current_line = line; } + while (opcode == EXTENDED_ARG) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + opcode = _Py_GetBaseOpcode(code, i); + } for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; @@ -1435,6 +1507,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_tools(code, i, event, new_tools); } } + while (base_opcode == EXTENDED_ARG) { + i++; + base_opcode = _Py_GetBaseOpcode(code, i); + } /* TO DO -- Use instruction length, not cache count */ if (base_opcode == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ @@ -1489,7 +1565,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - //sanity_check_instrumentation(code); + sanity_check_instrumentation(code); return 0; } @@ -1503,7 +1579,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static void instrument_all_executing_code_objects(PyInterpreterState *interp) { + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); while (ts) { _PyInterpreterFrame *frame = ts->cframe->current_frame; while (frame) { @@ -1512,7 +1591,9 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { } frame = frame->previous; } + HEAD_LOCK(runtime); ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } } diff --git a/Python/pystate.c b/Python/pystate.c index 30a30360778629..5c9339c312fa3d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -829,7 +829,6 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; - tstate->monitoring = 0; tstate->what_event = -1; tstate->_initialized = 1; From 8b8f67eae53fce0185a634d79e64b8e44d51e054 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 09:58:07 +0000 Subject: [PATCH 030/116] Make sure instrumentation respects instruction boundaries. --- Python/instrumentation.c | 99 ++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e92f1405687bcb..40c91ba4d7e75f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -337,13 +337,12 @@ Instruction read_instruction(PyCodeObject *code, int offset) result.deinstrumented_opcode = opcode; } int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; + /* TO DO -- Use instruction length, not cache count table */ + result.length += _PyOpcode_Caches[base]; if (base == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ result.length += 1; } - else { - result.length += _PyOpcode_Caches[base]; - } return result; } @@ -521,11 +520,17 @@ sanity_check_instrumentation(PyCodeObject *code) tools_matrix) ); int code_len = (int)Py_SIZE(code); - Instruction inst; for (int i = 0; i < code_len;) { - inst = read_instruction(code, i); - /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (inst.actual_opcode == INSTRUMENTED_LINE) { + Instruction inst = read_instruction(code, i); + int opcode = inst.actual_opcode; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = data->per_instruction_opcodes[i]; + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + /* TO DO -- check tools */ + } + if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); int opcode = data->lines[i].original_opcode; @@ -550,13 +555,20 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(tools_matrix.tools[event] != 0); CHECK(inst.deinstrumented_opcode != END_FOR); } + if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { + CHECK(_PyCode_CODE(code)[i+2].opcode == POP_JUMP_IF_FALSE || + _PyCode_CODE(code)[i+2].opcode == POP_JUMP_IF_TRUE); + } if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } CHECK(valid_opcode(inst.deinstrumented_opcode)); - if (inst.is_prefix_instrumented || inst.is_specialized_instrumented) { + if (inst.is_prefix_instrumented) { + CHECK(inst.extended_args || _PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); + } + if (inst.is_specialized_instrumented) { CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); } if (data->tools) { @@ -753,7 +765,6 @@ instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ if (opcode == INSTRUMENTED_INSTRUCTION) { return; } @@ -1308,39 +1319,34 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); + for (int i = code->_co_firsttraceable; i < code_len; ) { + Instruction inst = read_instruction(code, i); int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); - if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; - } - else { - line_data[i].original_opcode = 0; - } - switch (opcode) { + line_data[i].line_delta = compute_line_delta(code, i, line); + switch (inst.deinstrumented_opcode) { case END_ASYNC_FOR: case END_FOR: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; + break; + default: + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } + if (line >= 0) { + current_line = line; + } } - line_data[i].line_delta = compute_line_delta(code, i, line); - if (line >= 0) { - current_line = line; - } - while (opcode == EXTENDED_ARG) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - opcode = _Py_GetBaseOpcode(code, i); - } - for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; + for (int j = 1; j < inst.length; j++) { + line_data[i+j].original_opcode = 0; + line_data[i+j].line_delta = NO_LINE; } - switch (opcode) { + switch (inst.deinstrumented_opcode) { case RETURN_VALUE: case RAISE_VARARGS: case RERAISE: @@ -1348,6 +1354,7 @@ initialize_lines(PyCodeObject *code) * should be treated as different lines */ current_line = -1; } + i += inst.length; } } @@ -1525,7 +1532,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; - for (int i = code->_co_firsttraceable; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len;) { + Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1534,21 +1542,17 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_line_tools(code, i, new_line_tools); } } - int opcode = _Py_GetBaseOpcode(code, i); - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[opcode]; - if (opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - i++; - } + i += inst.length; } } uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (new_per_instruction_tools | removed_per_instruction_tools) { - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); + for (int i = code->_co_firsttraceable; i < code_len;) { + Instruction inst = read_instruction(code, i); + int opcode = _PyOpcode_Deopt[inst.deinstrumented_opcode]; if (opcode == RESUME || opcode == END_FOR) { + i += inst.length; continue; } if (removed_per_instruction_tools) { @@ -1557,15 +1561,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[opcode]; - if (opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - i++; - } + i += inst.length; } } +#ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); +#endif return 0; } From d0a2228ca5b008d2b87787c90dbaaff6b8f3b92a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 12:33:20 +0000 Subject: [PATCH 031/116] Fix handling of errors in jump instruments --- Lib/test/test_monitoring.py | 9 ++++--- Python/bytecodes.c | 50 ++++++++----------------------------- Python/generated_cases.c.h | 50 ++++++++----------------------------- 3 files changed, 28 insertions(+), 81 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1388a745b16992..11be2fe7fe1677 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -493,7 +493,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [493, 14, 494]) + start = LineMontoringTest.test_lines_single.__code__.co_firstlineno + self.assertEqual(events, [start+7, 14, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -510,7 +511,8 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 511]) + start = LineMontoringTest.test_lines_loop.__code__.co_firstlineno + self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -531,7 +533,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - expected = [530, 14, 531] + start = LineMontoringTest.test_lines_two.__code__.co_firstlineno + expected = [start+10, 14, start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a4566f574f393f..5bcbde155e2771 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3466,15 +3466,9 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - if (err) { - JUMPBY(oparg); - } - else { + int shrink = 1 - err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3486,16 +3480,10 @@ dummy_func( ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - if (err == 0) { - JUMPBY(oparg); - } - else { + _Py_CODEUNIT *target = next_instr + (1-err)*oparg; + int shrink = err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3508,12 +3496,7 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { @@ -3523,12 +3506,7 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) { @@ -3543,10 +3521,7 @@ dummy_func( Py_DECREF(value); offset = 0; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) { @@ -3561,10 +3536,7 @@ dummy_func( Py_DECREF(value); offset = oparg; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } // stack effect: ( -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c6fff9076bfc14..c91dd0fef592ea 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3789,15 +3789,9 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - if (err) { - JUMPBY(oparg); - } - else { + int shrink = 1 - err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3810,16 +3804,10 @@ if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - if (err == 0) { - JUMPBY(oparg); - } - else { + _Py_CODEUNIT *target = next_instr + (1-err)*oparg; + int shrink = err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3833,12 +3821,7 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3849,12 +3832,7 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3870,10 +3848,7 @@ Py_DECREF(value); offset = 0; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3889,10 +3864,7 @@ Py_DECREF(value); offset = oparg; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } From 2a3a85e8b80056a353fab818215ce6bfa1f5724f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 12:33:42 +0000 Subject: [PATCH 032/116] Fix memory leaks. --- Python/instrumentation.c | 12 ++++++++++-- Python/pystate.c | 8 +++++++- Python/sysmodule.c | 4 +++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 40c91ba4d7e75f..07a3330d232b93 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -9,7 +9,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -1068,6 +1068,7 @@ _Py_call_instrumentation_jump( ) { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); + assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; int offset = instr - _PyCode_CODE(code); @@ -1086,6 +1087,11 @@ _Py_call_instrumentation_jump( int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(to_obj); Py_DECREF(from_obj); + if (err) { + /* Error handling expects next_instr to point to instruction + 1. + * So we add one here. */ + frame->prev_instr++; + } return err; } @@ -1914,7 +1920,9 @@ PyObject *_Py_CreateMonitoringObject(void) if (events == NULL) { goto error; } - if (PyObject_SetAttrString(mod, "events", events)) { + int err = PyObject_SetAttrString(mod, "events", events); + Py_DECREF(events); + if (err) { goto error; } for (int i = 0; i < PY_MONITORING_EVENTS; i++) { diff --git a/Python/pystate.c b/Python/pystate.c index 5c9339c312fa3d..bcc48480aa98ab 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -421,6 +421,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); + for(int i = 0; i < PY_MONITORING_EVENTS; i++) { + interp->monitoring_matrix.tools[i] = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->tools[t].instrument_callables[i]); + } + } + PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); @@ -475,7 +482,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; - // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1291694f06d8b9..4c8cea23576a6d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3452,7 +3452,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (monitoring == NULL) { goto error; } - if (PyDict_SetItemString(sysdict, "monitoring", monitoring) < 0) { + int err = PyDict_SetItemString(sysdict, "monitoring", monitoring); + Py_DECREF(monitoring); + if (err < 0) { goto error; } From c3724ab3c1adeda46c3235fa058dce1e6d47433a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 16:51:35 +0000 Subject: [PATCH 033/116] Instrument FOR_ITER. --- Include/internal/pycore_opcode.h | 4 ++-- Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test__opcode.py | 8 ++++---- Python/bytecodes.c | 31 +++++++++++++++++++++++++++++-- Python/ceval.c | 11 ++--------- Python/generated_cases.c.h | 31 +++++++++++++++++++++++++++++-- Python/instrumentation.c | 7 +++++-- Python/legacy_tracing.c | 17 ++++++++++++++--- Python/opcode_metadata.h | 1 + Python/opcode_targets.h | 2 +- 11 files changed, 89 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index fb1359a93b8325..bbf777cf82f7de 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, @@ -481,7 +482,7 @@ static const char *const _PyOpcode_OpName[263] = { [234] = "<234>", [235] = "<235>", [236] = "<236>", - [237] = "<237>", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 234: \ case 235: \ case 236: \ - case 237: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 88ebea7c0efc20..c2c6faebe7f7a4 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define INSTRUMENTED_FOR_ITER 237 #define INSTRUMENTED_INSTRUCTION 238 #define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 diff --git a/Lib/opcode.py b/Lib/opcode.py index 7d4091ed1178bd..d1f95928a085bc 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,7 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions +def_op('INSTRUMENTED_FOR_ITER', 237) def_op('INSTRUMENTED_INSTRUCTION', 238) def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) def_op('INSTRUMENTED_RESUME', 240) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 5df05b6f57ded6..03850f68263a3a 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,8 +20,8 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 238: - # Opcodes 240 and up are internal instrumentation instructions + if code >= 237: + # Opcodes 237 and up are internal instrumentation instructions continue with self.subTest(opname=name): if code not in has_arg: @@ -54,8 +54,8 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 238: - # Opcodes 240 and up are internal instrumentation instructions + if code >= 237: + # Opcodes 237 and up are internal instrumentation instructions continue with self.subTest(opname=name): if code not in has_arg: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5bcbde155e2771..f773ae2ad27d91 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -129,10 +129,9 @@ dummy_func( else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg != 0, frame, next_instr-1); + tstate, oparg > 0, frame, next_instr-1); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); /* Instrumentation has jumped */ next_instr = frame->prev_instr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2323,6 +2322,34 @@ dummy_func( } } + // stack effect: ( -- __0) + inst(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + } + // stack effect: ( -- __0) inst(FOR_ITER_LIST) { _PyListIterObject *it = (_PyListIterObject *)TOP(); diff --git a/Python/ceval.c b/Python/ceval.c index 8716ab82dad5bb..18f044f6fbb80a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2255,22 +2255,15 @@ monitor_raise(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } - PyObject *type, *value, *traceback, *orig_traceback, *arg; + PyObject *type, *value, *orig_traceback; int err; _PyErr_Fetch(tstate, &type, &value, &orig_traceback); if (value == NULL) { value = Py_NewRef(Py_None); } _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; assert(value != NULL && value != Py_None); - arg = PyTuple_Pack(3, type, value, traceback); - if (arg == NULL) { - _PyErr_Restore(tstate, type, value, orig_traceback); - return; - } - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, arg); - Py_DECREF(arg); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, value); if (err == 0) { _PyErr_Restore(tstate, type, value, orig_traceback); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c91dd0fef592ea..b21057b8e94998 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -18,10 +18,9 @@ else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg != 0, frame, next_instr-1); + tstate, oparg > 0, frame, next_instr-1); if (err) goto error; if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); /* Instrumentation has jumped */ next_instr = frame->prev_instr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2632,6 +2631,34 @@ DISPATCH(); } + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + TARGET(FOR_ITER_LIST) { _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 07a3330d232b93..c6adcdd1c6c805 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -45,6 +45,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -76,6 +78,7 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = true, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, [INSTRUMENTED_COMPARE_AND_BRANCH] = true, + [INSTRUMENTED_FOR_ITER] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -93,6 +96,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, + [INSTRUMENTED_FOR_ITER] = FOR_ITER, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -1504,8 +1508,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; if (base_opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg > 0; + event = instr->oparg > 0; } else { event = EVENT_FOR_OPCODE[base_opcode]; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 22af735304df4a..12580036036cb1 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -116,9 +116,20 @@ sys_trace_exception_func( ) { assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); - PyObject *arg = args[2]; - assert(PyTuple_CheckExact(arg)); - PyObject *res = call_trace_func(self, arg); + PyObject *exc = args[2]; + assert(PyExceptionInstance_Check(exc)); + PyObject *type = (PyObject *)Py_TYPE(exc); + PyObject *tb = PyException_GetTraceback(exc); + if (tb == NULL) { + tb = Py_NewRef(Py_None); + } + PyObject * tuple = PyTuple_Pack(3, type, exc, tb); + Py_DECREF(tb); + if (tuple == NULL) { + return NULL; + } + PyObject *res = call_trace_func(self, tuple); + Py_DECREF(tuple); return res; } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 9ad33884808bf7..41dc9ef0ba0d71 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -144,6 +144,7 @@ static const struct { [GET_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_YIELD_FROM_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [FOR_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [INSTRUMENTED_FOR_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_LIST] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_TUPLE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_RANGE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 0c4ea0ec2b4a3d..113b664fffccf1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -236,7 +236,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, &&TARGET_INSTRUMENTED_RESUME, From d64823c79598fccf159eef3e7bca3d23c27f5f0a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 2 Feb 2023 18:54:36 +0000 Subject: [PATCH 034/116] Fix legacy profiling of calls. --- Include/internal/pycore_instruments.h | 12 +++++- Lib/test/test_monitoring.py | 4 +- Modules/_lsprof.c | 4 ++ Python/bytecodes.c | 34 +++++++++------- Python/ceval.c | 4 +- Python/generated_cases.c.h | 34 +++++++++------- Python/instrumentation.c | 56 ++++++++++++++++++++++++--- Python/legacy_tracing.c | 26 +++++++++++-- 8 files changed, 131 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 6c76027889457d..a64633aa1edb47 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -83,9 +83,17 @@ extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +extern int +_Py_call_instrumentation_2args(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + extern void -_Py_call_instrumentation_exc(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern void +_Py_call_instrumentation_exc2(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 11be2fe7fe1677..c6a813fe4b55ae 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -206,7 +206,7 @@ def gather_events(self, func): def record(*args, event_name=event_name): events.append(event_name) sys.monitoring.register_callback(TEST_TOOL, event, record) - def record_call(code, offset, obj): + def record_call(code, offset, obj, arg): if isinstance(obj, PY_CALLABLES): events.append("py_call") else: @@ -282,7 +282,7 @@ def down(*args): frame = sys._getframe(1) stack.append(frame) seen.add(frame.f_code) - def call(code, offset, callable): + def call(code, offset, callable, arg): if not isinstance(callable, PY_CALLABLES): stack.append(sys._getframe(1)) for event in UP_EVENTS: diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 37170bbea56ad3..d0a483e6b1d704 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -126,6 +126,10 @@ call_timer(ProfilerObject *pObj) static PyObject * normalizeUserObj(PyObject *obj) { + //if (Py_TYPE(obj) == &PyMethodDescr_Type) { + // assert(0); + // return PyObject_Repr(obj); + //} PyCFunctionObject *fn; if (!PyCFunction_Check(obj)) { return Py_NewRef(obj); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f773ae2ad27d91..f97a14b17c8a71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2335,9 +2335,9 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + monitor_raise(tstate, frame, here); goto error; } - monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2634,9 +2634,10 @@ dummy_func( int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - int err = _Py_call_instrumentation_arg( + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); ERROR_IF(err, error); _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -2699,15 +2700,16 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); if (res == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err < 0) { Py_CLEAR(res); } @@ -3236,21 +3238,25 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { - int err = _Py_call_instrumentation_arg( + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); ERROR_IF(err, error); result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err < 0) { Py_CLEAR(result); } diff --git a/Python/ceval.c b/Python/ceval.c index 18f044f6fbb80a..d7e4a9f9a3049e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2283,7 +2283,7 @@ monitor_unwind(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { return; } - _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); } @@ -2306,7 +2306,7 @@ monitor_throw(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { return; } - _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr, NULL); + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); } void diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b21057b8e94998..eb42b76d2e193b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2643,9 +2643,9 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + monitor_raise(tstate, frame, here); goto error; } - monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2948,9 +2948,10 @@ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - int err = _Py_call_instrumentation_arg( + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err) goto error; _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3013,15 +3014,16 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); if (res == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err < 0) { Py_CLEAR(res); } @@ -3550,21 +3552,25 @@ } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { - int err = _Py_call_instrumentation_arg( + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err < 0) { Py_CLEAR(result); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c6adcdd1c6c805..ee62afb103a913 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1065,6 +1065,26 @@ _Py_call_instrumentation_arg( return err; } +int +_Py_call_instrumentation_2args( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + PyCodeObject *code = frame->f_code; + assert(is_instrumentation_up_to_date(code, tstate->interp)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { + return -1; + } + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[5] = { NULL, (PyObject *)code, offset_obj, arg0, arg1 }; + int err = call_instrument(tstate, code, event, &args[1], 4 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); + return err; +} + int _Py_call_instrumentation_jump( PyThreadState *tstate, int event, @@ -1100,26 +1120,29 @@ _Py_call_instrumentation_jump( } void -_Py_call_instrumentation_exc( +_Py_call_instrumentation_exc_vector( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); + assert(args[0] == NULL); PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; + assert(args[1] == NULL); + args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); + uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { err = -1; } else { - PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; - Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(args[2] == NULL); + args[2] = offset_obj; err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); Py_DECREF(offset_obj); } @@ -1134,6 +1157,29 @@ _Py_call_instrumentation_exc( assert(_PyErr_Occurred(tstate)); } +void +_Py_call_instrumentation_exc0( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[3] = { NULL, NULL, NULL }; + Py_ssize_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); +} + +void +_Py_call_instrumentation_exc2( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + Py_ssize_t nargsf = 4 | PY_VECTORCALL_ARGUMENTS_OFFSET; + _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); +} + + int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 12580036036cb1..f494134d9340c0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -79,12 +79,30 @@ sys_profile_call_or_return( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - assert(PyVectorcall_NARGS(nargsf) == 3); + assert(PyVectorcall_NARGS(nargsf) == 4); PyObject *callable = args[2]; - if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { - Py_RETURN_NONE; + if (PyCFunction_Check(callable)) { + return call_profile_func(self, callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + PyObject *self_arg = args[3]; + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == Py_None) { + Py_RETURN_NONE; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + PyObject *res = call_profile_func(self, meth); + Py_DECREF(meth); + return res; } - return call_profile_func(self, callable); + Py_RETURN_NONE; } static PyObject * From 284d0b113045c5d7f43e6474433cbc6a1e889fad Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 3 Feb 2023 15:10:34 +0000 Subject: [PATCH 035/116] Fix up instrumented yield. --- Python/bytecodes.c | 23 ++++++++++++++++------- Python/generated_cases.c.h | 25 +++++++++++++++++-------- Python/opcode_metadata.h | 2 +- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f97a14b17c8a71..ec05dff27b18fc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -768,14 +768,24 @@ dummy_func( } } - inst(INSTRUMENTED_YIELD_VALUE, ( -- )) { - PyObject *val = TOP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, val); + frame, next_instr-1, retval); ERROR_IF(err, error); - GO_TO_INSTRUCTION(YIELD_VALUE); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } inst(YIELD_VALUE, (retval -- unused)) { @@ -2694,8 +2704,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; - res = PyObject_Vectorcall( + PyObject *res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eb42b76d2e193b..f242d9cfce51c4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -934,17 +934,27 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *val = TOP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *retval = PEEK(1); + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, val); - if (err) goto error; - GO_TO_INSTRUCTION(YIELD_VALUE); + frame, next_instr-1, retval); + if (err) goto pop_1_error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } TARGET(YIELD_VALUE) { - PREDICTED(YIELD_VALUE); PyObject *retval = PEEK(1); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -3008,8 +3018,7 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; - res = PyObject_Vectorcall( + PyObject *res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 41dc9ef0ba0d71..e7b5bde0c0475f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -61,7 +61,7 @@ static const struct { [GET_ANEXT] = { 1, 2, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_AWAITABLE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [SEND] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [INSTRUMENTED_YIELD_VALUE] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [POP_EXCEPT] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 1440473a071784b0753420dccb7ee0171253b248 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Feb 2023 10:33:39 +0000 Subject: [PATCH 036/116] Fix instrumentation of SEND and tidy up instrumented bytecode numbering. --- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 6 +-- Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 8 ++-- Lib/test/test_sys_settrace.py | 3 -- Python/bytecodes.c | 27 +---------- Python/ceval.c | 17 ++----- Python/generated_cases.c.h | 27 +---------- Python/legacy_tracing.c | 71 ++++++++++++++++++++++++++-- Python/opcode_targets.h | 2 +- 10 files changed, 88 insertions(+), 79 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 42c3cbcb5e6496..af565176279c79 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -495,10 +495,10 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = "INSTRUMENTED_JUMP_IF_FALSE_OR_POP", - [248] = "<248>", [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [251] = "<251>", [252] = "<252>", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", @@ -580,7 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 248: \ + case 251: \ case 252: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index c2068edc908388..3eda340f1c4d24 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -130,9 +130,9 @@ extern "C" { #define INSTRUMENTED_JUMP_FORWARD 245 #define INSTRUMENTED_JUMP_BACKWARD 246 #define INSTRUMENTED_JUMP_IF_FALSE_OR_POP 247 -#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 249 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 250 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 251 +#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 347bda99054101..a7cc58ca3d6a32 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -446,7 +446,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 879291569cb32f..71e5b43a67c697 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -240,13 +240,13 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) -def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 249) -def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 250) -def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) +def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) +def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) +def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) -# 255 will be MARKER +# 255 is reserved hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..88a2b941b99ad4 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -636,21 +636,18 @@ async def f(): (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e2c3504a643d86..4a92c1a65dc402 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -733,31 +733,8 @@ dummy_func( inst(SEND, (receiver, v -- receiver if (!jump), retval)) { assert(frame != &entry_frame); bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, next_instr-1); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } - } - else { - gen_status = PYGEN_NEXT; - } - } + + PySendResult gen_status = PyIter_Send(receiver, v, &retval); if (gen_status == PYGEN_ERROR) { assert(retval == NULL); goto error; diff --git a/Python/ceval.c b/Python/ceval.c index 4813342d9173b8..fe6a7eb019c7c1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1949,22 +1949,15 @@ monitor_raise(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } - PyObject *type, *value, *orig_traceback; + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); int err; - _PyErr_Fetch(tstate, &type, &value, &orig_traceback); - if (value == NULL) { - value = Py_NewRef(Py_None); - } - _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - assert(value != NULL && value != Py_None); - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, value); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, exc); if (err == 0) { - _PyErr_Restore(tstate, type, value, orig_traceback); + PyErr_SetRaisedException(exc); } else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(orig_traceback); + Py_DECREF(exc); } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3bfa7c627b985f..21f25bf0589e94 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -938,31 +938,8 @@ PyObject *retval; assert(frame != &entry_frame); bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, next_instr-1); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } - } - else { - gen_status = PYGEN_NEXT; - } - } + + PySendResult gen_status = PyIter_Send(receiver, v, &retval); if (gen_status == PYGEN_ERROR) { assert(retval == NULL); goto error; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index f494134d9340c0..140ae4ec8bb94a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -105,6 +105,8 @@ sys_profile_call_or_return( Py_RETURN_NONE; } + + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -162,7 +164,65 @@ sys_trace_func2( } static PyObject * -sys_trace_func3( +sys_trace_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(!PyErr_Occurred()); + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + assert(PyCode_Check(args[0])); + PyCodeObject *code = (PyCodeObject *)args[0]; + PyObject *val = args[2]; + PyObject *res = call_trace_func(self, val); + if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && + res != NULL + ) { + /* Fake PEP 380 StopIteration exception event */ + Py_DECREF(res); + PyObject *stop_iter_args[2] = { NULL, val }; + PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, + &stop_iter_args[1], + 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (exc == NULL) { + return NULL; + } + /* The event occurs in the caller frame, for historical reasons */ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + frame = frame->previous; + while (frame && _PyFrame_IsIncomplete(frame)) { + frame = frame->previous; + } + PyFrameObject* frame_obj = NULL; + if (frame != NULL) { + frame_obj = _PyFrame_GetFrameObject(frame); + } else { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + if (frame_obj == NULL) { + Py_DECREF(exc); + return NULL; + } + Py_INCREF(frame_obj); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, exc); + Py_DECREF(exc); + Py_DECREF(frame_obj); + if (err) { + return NULL; + } + } + return res; +} + +static PyObject * +sys_trace_yield( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -434,8 +494,13 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + (vectorcallfunc)sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c7f985981689a5..6112dc790f4553 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -247,11 +247,11 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, &&TARGET_DO_TRACING From 825f42a294ddda8a68e8eae5d708fd3e291d5672 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Feb 2023 15:09:32 +0000 Subject: [PATCH 037/116] Handle line numbers in exception handled event. --- Lib/test/test_bdb.py | 3 +- Lib/test/test_monitoring.py | 58 +++++++++++++- Lib/test/test_sys_settrace.py | 3 + Python/bytecodes.c | 32 ++++---- Python/generated_cases.c.h | 32 ++++---- Python/legacy_tracing.c | 147 +++++++++++++++++++++------------- 6 files changed, 191 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 87a5ac308a12df..9f2f964f7f54bb 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -433,8 +433,9 @@ def __exit__(self, type_=None, value=None, traceback=None): not_empty = '' if self.tracer.set_list: not_empty += 'All paired tuples have not been processed, ' - not_empty += ('the last one was number %d' % + not_empty += ('the last one was number %d\n' % self.tracer.expect_set_no) + not_empty += repr(self.tracer.set_list) # Make a BdbNotExpectedError a unittest failure. if type_ is not None and issubclass(BdbNotExpectedError, type_): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index c6a813fe4b55ae..27308f9a6fc1ad 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -538,13 +538,69 @@ def test_lines_two(self): self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: - sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() + def check_lines(self, func, expected, tool=TEST_TOOL): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(tool, E.LINE, recorder) + sys.monitoring.set_events(tool, E.LINE) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, E.LINE, None) + lines = [ line - func.__code__.co_firstlineno for line in events[1:-1] ] + self.assertEqual(lines, expected) + finally: + sys.monitoring.set_events(tool, 0) + + + def test_linear(self): + + def func(): + line = 1 + line = 2 + line = 3 + line = 4 + line = 5 + + self.check_lines(func, [1,2,3,4,5]) + + def test_branch(self): + def func(): + if "true".startswith("t"): + line = 2 + line = 3 + else: + line = 5 + line = 6 + + self.check_lines(func, [1,2,3,6]) + + def test_try_except(self): + + def func1(): + try: + line = 2 + line = 3 + except: + line = 5 + line = 6 + self.check_lines(func1, [1,2,3,6]) + def func2(): + try: + line = 2 + raise 3 + except: + line = 5 + line = 6 + self.check_lines(func2, [1,2,3,4,5,6]) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 88a2b941b99ad4..a251b2272e95eb 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -636,18 +636,21 @@ async def f(): (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4a92c1a65dc402..8763cf82a79b03 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -733,22 +733,26 @@ dummy_func( inst(SEND, (receiver, v -- receiver if (!jump), retval)) { assert(frame != &entry_frame); bool jump = false; - - PySendResult gen_status = PyIter_Send(receiver, v, &retval); - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { - assert(gen_status == PYGEN_NEXT); - assert(retval != NULL); + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + Py_DECREF(receiver); + JUMPBY(oparg); + jump = true; + } + else { + goto error; + } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 21f25bf0589e94..cc5ff8ec3c5a82 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -938,22 +938,26 @@ PyObject *retval; assert(frame != &entry_frame); bool jump = false; - - PySendResult gen_status = PyIter_Send(receiver, v, &retval); - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { - assert(gen_status == PYGEN_NEXT); - assert(retval != NULL); + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + Py_DECREF(receiver); + JUMPBY(oparg); + jump = true; + } + else { + goto error; + } } STACK_SHRINK(1); STACK_GROW(((!jump) ? 1 : 0)); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 140ae4ec8bb94a..12183820bb0d35 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -169,9 +169,6 @@ sys_trace_return( size_t nargsf, PyObject *kwnames ) { PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { - Py_RETURN_NONE; - } assert(!PyErr_Occurred()); assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); @@ -179,45 +176,57 @@ sys_trace_return( PyCodeObject *code = (PyCodeObject *)args[0]; PyObject *val = args[2]; PyObject *res = call_trace_func(self, val); - if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && - res != NULL - ) { - /* Fake PEP 380 StopIteration exception event */ - Py_DECREF(res); - PyObject *stop_iter_args[2] = { NULL, val }; - PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, - &stop_iter_args[1], - 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (exc == NULL) { - return NULL; - } - /* The event occurs in the caller frame, for historical reasons */ - _PyInterpreterFrame *frame = _PyEval_GetFrame(); - frame = frame->previous; - while (frame && _PyFrame_IsIncomplete(frame)) { - frame = frame->previous; - } - PyFrameObject* frame_obj = NULL; - if (frame != NULL) { - frame_obj = _PyFrame_GetFrameObject(frame); - } else { - PyErr_SetString(PyExc_SystemError, - "Missing frame when calling trace function."); - return NULL; - } - if (frame_obj == NULL) { - Py_DECREF(exc); - return NULL; - } - Py_INCREF(frame_obj); - int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, exc); - Py_DECREF(exc); - Py_DECREF(frame_obj); - if (err) { - return NULL; - } - } +// if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && +// res != NULL && tstate->c_tracefunc != NULL +// ) { +// /* Fake PEP 380 StopIteration exception event */ +// Py_DECREF(res); +// PyObject *stop_iter_args[2] = { NULL, val }; +// PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, +// &stop_iter_args[1], +// 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, +// NULL); +// if (exc == NULL) { +// return NULL; +// } +// /* The event occurs in the caller frame, for historical reasons */ +// _PyInterpreterFrame *frame = _PyEval_GetFrame(); +// frame = frame->previous; +// while (frame && _PyFrame_IsIncomplete(frame)) { +// frame = frame->previous; +// } +// PyFrameObject* frame_obj = NULL; +// if (frame != NULL) { +// frame_obj = _PyFrame_GetFrameObject(frame); +// } else { +// PyErr_SetString(PyExc_SystemError, +// "Missing frame when calling trace function."); +// return NULL; +// } +// if (frame_obj == NULL) { +// Py_DECREF(exc); +// return NULL; +// } +// PyObject *type = (PyObject *)Py_TYPE(exc); +// PyObject *tb = PyException_GetTraceback(exc); +// if (tb == NULL) { +// tb = Py_NewRef(Py_None); +// } +// PyObject * tuple = PyTuple_Pack(3, type, exc, tb); +// Py_DECREF(tb); +// if (tuple == NULL) { +// Py_DECREF(exc); +// return NULL; +// } +// Py_INCREF(frame_obj); +// int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, tuple); +// Py_DECREF(frame_obj); +// Py_DECREF(tuple); +// Py_DECREF(exc); +// if (err) { +// return NULL; +// } +// } return res; } @@ -263,6 +272,9 @@ trace_line( PyThreadState *tstate, _PyLegacyEventHandler *self, PyFrameObject* frame, int line ) { + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } if (line < 0) { Py_RETURN_NONE; } @@ -298,9 +310,6 @@ sys_trace_line_func( return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); - if (!frame->f_trace_lines) { - Py_RETURN_NONE; - } if (frame ->f_last_traced_line == line) { /* Already traced this line */ Py_RETURN_NONE; @@ -341,19 +350,44 @@ sys_trace_jump_func( int to_line = _Py_Instrumentation_GetLine(code, to); /* Backward jump: Always generate event * Forward jump: Only generate event if jumping to different line. */ - if (to > from) { - /* Forwards jump */ - if (frame->f_last_traced_line == to_line) { - /* Already traced this line */ - Py_RETURN_NONE; - } + if (to > from && frame->f_last_traced_line == to_line) { + /* Already traced this line */ + Py_RETURN_NONE; } return trace_line(tstate, self, frame, to_line); } +/* We don't care about the exception here, + * we just treat it as a possible new line + */ +static PyObject * +sys_trace_exception_handled( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(PyVectorcall_NARGS(nargsf) == 3); + PyFrameObject* frame = PyEval_GetFrame(); + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); + int offset = _PyLong_AsInt(args[1]); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int line = _Py_Instrumentation_GetLine(code, offset); + if (frame->f_last_traced_line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, self, frame, line); +} + PyTypeObject _PyLegacyEventHandler_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), @@ -528,7 +562,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, + PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { + return -1; + } } int delta = (func != NULL) - (tstate->c_tracefunc != NULL); @@ -545,7 +583,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } From ce5ddb32aa5f7d3ad7fe6858aaf36ebdf5210473 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 Feb 2023 09:50:48 +0000 Subject: [PATCH 038/116] Instrument END_FOR to mimic PEP 380 StopIteration and add more tests. --- Include/internal/pycore_instruments.h | 10 +++--- Include/internal/pycore_opcode.h | 4 +-- Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test_monitoring.py | 49 +++++++++++++++++++++++++++ Python/bytecodes.c | 24 +++++++++---- Python/ceval.c | 10 +++--- Python/generated_cases.c.h | 29 ++++++++++++---- Python/instrumentation.c | 9 ++++- Python/legacy_tracing.c | 3 +- Python/opcode_metadata.h | 5 +++ Python/opcode_targets.h | 2 +- Python/specialize.c | 6 ++-- 13 files changed, 125 insertions(+), 28 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index a64633aa1edb47..34dd2551f8aaa6 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -25,20 +25,22 @@ extern "C" { #define PY_MONITORING_EVENT_INSTRUCTION 6 #define PY_MONITORING_EVENT_JUMP 7 #define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 -#define PY_MONITORING_INSTRUMENTED_EVENTS 9 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 /* Grouped events */ -#define PY_MONITORING_EVENT_C_RETURN 9 -#define PY_MONITORING_EVENT_C_RAISE 10 +#define PY_MONITORING_EVENT_C_RETURN 10 +#define PY_MONITORING_EVENT_C_RAISE 11 /* Exceptional events */ -#define PY_MONITORING_EVENT_PY_THROW 11 #define PY_MONITORING_EVENT_RAISE 12 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 #define PY_MONITORING_EVENT_PY_UNWIND 14 +#define PY_MONITORING_EVENT_PY_THROW 15 + /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index af565176279c79..b2d8e57b3c8024 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -498,7 +499,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [251] = "<251>", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", [252] = "<252>", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 251: \ case 252: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 3eda340f1c4d24..8dc6ef9655760d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -133,6 +133,7 @@ extern "C" { #define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 248 #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 diff --git a/Lib/opcode.py b/Lib/opcode.py index 71e5b43a67c697..022a8885eef43a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -243,6 +243,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) +def_op('INSTRUMENTED_END_FOR', 251) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 27308f9a6fc1ad..37961cc981fb83 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -604,3 +604,52 @@ def func2(): line = 6 self.check_lines(func2, [1,2,3,4,5,6]) + + +class ExceptionRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, exc): + self.events.append(type(exc)) + +class ExceptionMontoringTest(unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, events=E.RAISE): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + recorder = ExceptionRecorder(event_list) + sys.monitoring.register_callback(tool, events, recorder) + sys.monitoring.set_events(tool, events) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, events, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, events, None) + + + def test_simple_try_except(self): + + def func1(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func1, [KeyError]) + + def gen(): + yield 1 + return 2 + + def implicit_stop_iteration(): + for _ in gen(): + pass + + self.check_events(implicit_stop_iteration, [StopIteration], events=E.STOP_ITERATION) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8763cf82a79b03..4ac79ea25acacb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,6 +172,15 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; + inst(INSTRUMENTED_END_FOR, (pop1, pop2 --)) { + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + DECREF_INPUTS(); + } + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -742,7 +751,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2147,11 +2156,12 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2173,13 +2183,14 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, here); goto error; } + monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -2272,7 +2283,8 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->opcode == END_FOR || + next_instr->opcode == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); } diff --git a/Python/ceval.c b/Python/ceval.c index fe6a7eb019c7c1..5a870eee02c107 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -178,7 +178,7 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, int event); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -905,7 +905,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); exception_unwind: { @@ -1944,15 +1944,15 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, int event) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { + if (tstate->interp->monitoring_matrix.tools[event] == 0) { return; } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err; - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, exc); + err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); if (err == 0) { PyErr_SetRaisedException(exc); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc5ff8ec3c5a82..ceea168c763258 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -235,6 +235,20 @@ DISPATCH(); } + TARGET(INSTRUMENTED_END_FOR) { + PyObject *pop2 = PEEK(1); + PyObject *pop1 = PEEK(2); + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + Py_DECREF(pop1); + Py_DECREF(pop2); + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(UNARY_NEGATIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -947,7 +961,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2711,11 +2725,12 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2741,13 +2756,14 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, here); goto error; } + monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -2852,7 +2868,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->opcode == END_FOR || + next_instr->opcode == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 25a38d70e492fb..3c5750c4b40aa2 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -49,6 +49,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -83,6 +85,8 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, [INSTRUMENTED_COMPARE_AND_BRANCH] = true, [INSTRUMENTED_FOR_ITER] = true, + [END_FOR] = true, + [INSTRUMENTED_END_FOR] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -102,6 +106,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, + [INSTRUMENTED_END_FOR] = END_FOR, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -135,6 +140,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, @@ -1909,7 +1916,7 @@ const char *event_names[] = { [PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED", [PY_MONITORING_EVENT_C_RAISE] = "C_RAISE", [PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND", - [15] = "Unused", + [PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION", }; /*[clinic input] diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 12183820bb0d35..d9f4b4adfa48e0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -539,7 +539,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, - PY_MONITORING_EVENT_RAISE, -1)) { + PY_MONITORING_EVENT_RAISE, PY_MONITORING_EVENT_STOP_ITERATION)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -584,6 +584,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION) | (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 2ea8b1027cbc80..bcc628f0ab05e2 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -38,6 +38,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case END_FOR: return 1+1; + case INSTRUMENTED_END_FOR: + return 2; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -422,6 +424,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case END_FOR: return 0+0; + case INSTRUMENTED_END_FOR: + return 0; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -794,6 +798,7 @@ struct opcode_metadata { [POP_TOP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [PUSH_NULL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [INSTRUMENTED_END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NEGATIVE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_INVERT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6112dc790f4553..874231dc5dc12f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -250,7 +250,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_FOR, &&_unknown_opcode, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, diff --git a/Python/specialize.c b/Python/specialize.c index 1bf90f429268fb..940de62e33d51e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1084,7 +1084,7 @@ PyObject *descr, DescriptorClassification kind) if (dict) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; - } + } assert(owner_cls->tp_dictoffset > 0); assert(owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); @@ -2170,7 +2170,9 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].opcode == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].opcode == INSTRUMENTED_END_FOR + ); _py_set_opcode(instr, FOR_ITER_GEN); goto success; } From da83abead70680e9f96523aae8529ec53f070a44 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 12:30:52 +0000 Subject: [PATCH 039/116] Add END_SEND for instrumentation. --- Include/internal/pycore_opcode.h | 44 +++++------ Include/opcode.h | 126 ++++++++++++++++--------------- Lib/opcode.py | 2 + Python/bytecodes.c | 41 ++++++++-- Python/compile.c | 3 +- Python/generated_cases.c.h | 57 +++++++++++--- Python/instrumentation.c | 7 ++ Python/legacy_tracing.c | 4 + Python/opcode_metadata.h | 10 +++ Python/opcode_targets.h | 40 +++++----- 10 files changed, 213 insertions(+), 121 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index e3b9e03b6f3e25..f566b2673c5f26 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -122,6 +122,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [END_SEND] = END_SEND, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -141,6 +142,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -255,17 +257,18 @@ static const char *const _PyOpcode_OpName[263] = { [PUSH_NULL] = "PUSH_NULL", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -274,20 +277,20 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -298,7 +301,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_AND_BRANCH_FLOAT] = "COMPARE_AND_BRANCH_FLOAT", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -306,38 +308,38 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [COMPARE_AND_BRANCH_FLOAT] = "COMPARE_AND_BRANCH_FLOAT", [COMPARE_AND_BRANCH_INT] = "COMPARE_AND_BRANCH_INT", [COMPARE_AND_BRANCH_STR] = "COMPARE_AND_BRANCH_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", @@ -363,7 +365,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -393,7 +395,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [COMPARE_AND_BRANCH] = "COMPARE_AND_BRANCH", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -403,21 +405,21 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [167] = "<167>", [168] = "<168>", [169] = "<169>", [170] = "<170>", @@ -502,7 +504,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [252] = "<252>", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [DO_TRACING] = "DO_TRACING", @@ -517,7 +519,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 167: \ case 168: \ case 169: \ case 170: \ @@ -582,7 +583,6 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 252: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index cd10052b7aadf2..efd91df2d730cd 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -13,6 +13,7 @@ extern "C" { #define PUSH_NULL 2 #define INTERPRETER_EXIT 3 #define END_FOR 4 +#define END_SEND 5 #define NOP 9 #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 @@ -134,6 +135,7 @@ extern "C" { #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 @@ -145,68 +147,68 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 10 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_AND_BRANCH_FLOAT 48 -#define COMPARE_AND_BRANCH_INT 56 -#define COMPARE_AND_BRANCH_STR 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_TUPLE 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 63 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_CONST__LOAD_FAST 79 -#define LOAD_FAST__LOAD_CONST 80 -#define LOAD_FAST__LOAD_FAST 81 -#define LOAD_GLOBAL_BUILTIN 82 -#define LOAD_GLOBAL_MODULE 84 -#define STORE_ATTR_INSTANCE_VALUE 86 -#define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 153 -#define STORE_SUBSCR_DICT 154 -#define STORE_SUBSCR_LIST_INT 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 160 -#define UNPACK_SEQUENCE_TWO_TUPLE 161 -#define SEND_GEN 166 +#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_ADD_INT 7 +#define BINARY_OP_ADD_UNICODE 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 10 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_PY_EXACT_ARGS 22 +#define CALL_PY_WITH_DEFAULTS 23 +#define CALL_BOUND_METHOD_EXACT_ARGS 24 +#define CALL_BUILTIN_CLASS 28 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 +#define CALL_NO_KW_BUILTIN_FAST 38 +#define CALL_NO_KW_BUILTIN_O 39 +#define CALL_NO_KW_ISINSTANCE 40 +#define CALL_NO_KW_LEN 41 +#define CALL_NO_KW_LIST_APPEND 42 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 +#define CALL_NO_KW_STR_1 46 +#define CALL_NO_KW_TUPLE_1 47 +#define CALL_NO_KW_TYPE_1 48 +#define COMPARE_AND_BRANCH_FLOAT 56 +#define COMPARE_AND_BRANCH_INT 57 +#define COMPARE_AND_BRANCH_STR 58 +#define FOR_ITER_LIST 59 +#define FOR_ITER_TUPLE 62 +#define FOR_ITER_RANGE 63 +#define FOR_ITER_GEN 64 +#define LOAD_ATTR_CLASS 65 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_PROPERTY 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_WITH_HINT 76 +#define LOAD_ATTR_METHOD_LAZY_DICT 77 +#define LOAD_ATTR_METHOD_NO_DICT 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_CONST__LOAD_FAST 80 +#define LOAD_FAST__LOAD_CONST 81 +#define LOAD_FAST__LOAD_FAST 82 +#define LOAD_GLOBAL_BUILTIN 84 +#define LOAD_GLOBAL_MODULE 86 +#define STORE_ATTR_INSTANCE_VALUE 87 +#define STORE_ATTR_SLOT 113 +#define STORE_ATTR_WITH_HINT 143 +#define STORE_FAST__LOAD_FAST 153 +#define STORE_FAST__STORE_FAST 154 +#define STORE_SUBSCR_DICT 158 +#define STORE_SUBSCR_LIST_INT 159 +#define UNPACK_SEQUENCE_LIST 160 +#define UNPACK_SEQUENCE_TUPLE 161 +#define UNPACK_SEQUENCE_TWO_TUPLE 166 +#define SEND_GEN 167 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 582b008bc28e1d..55756c552ad1de 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -83,6 +83,7 @@ def pseudo_op(name, op, real_ops): def_op('INTERPRETER_EXIT', 3) def_op('END_FOR', 4) +def_op('END_SEND', 5) def_op('NOP', 9) @@ -244,6 +245,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) +def_op('INSTRUMENTED_END_SEND', 252) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4ba7be06429435..5b1ee975cb0944 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,15 +172,30 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; - inst(INSTRUMENTED_END_FOR, (pop1, pop2 --)) { + inst(INSTRUMENTED_END_FOR, (receiver, discard --)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - PyErr_SetNone(PyExc_StopIteration); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); - PyErr_SetRaisedException(NULL); + if (PyGen_Check(receiver)) { + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } DECREF_INPUTS(); } + inst(END_SEND, (receiver, value -- value)) { + Py_DECREF(receiver); + } + + inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + } + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -756,6 +771,20 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -766,10 +795,10 @@ dummy_func( if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); - assert(retval != NULL); - JUMPBY(oparg); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { goto error; diff --git a/Python/compile.c b/Python/compile.c index b49eda314eeef1..a1771b77ceb3a8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1789,8 +1789,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, POP_TOP); + ADDOP(c, loc, END_SEND); return SUCCESS; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7470803dd52a03..1c1e469aaa19fd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -236,19 +236,44 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *pop2 = PEEK(1); - PyObject *pop1 = PEEK(2); + PyObject *discard = PEEK(1); + PyObject *receiver = PEEK(2); /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - PyErr_SetNone(PyExc_StopIteration); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); - PyErr_SetRaisedException(NULL); - Py_DECREF(pop1); - Py_DECREF(pop2); + if (PyGen_Check(receiver)) { + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + Py_DECREF(discard); STACK_SHRINK(2); DISPATCH(); } + TARGET(END_SEND) { + PyObject *value = PEEK(1); + PyObject *receiver = PEEK(2); + Py_DECREF(receiver); + STACK_SHRINK(1); + POKE(1, value); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + PyObject *value = PEEK(1); + PyObject *receiver = PEEK(2); + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + STACK_SHRINK(1); + POKE(1, value); + DISPATCH(); + } + TARGET(UNARY_NEGATIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -962,6 +987,20 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -972,10 +1011,10 @@ if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); - assert(retval != NULL); - JUMPBY(oparg); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { goto error; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c5750c4b40aa2..ab254aff29679a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -51,6 +51,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -87,6 +89,8 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_FOR_ITER] = true, [END_FOR] = true, [INSTRUMENTED_END_FOR] = true, + [END_SEND] = true, + [INSTRUMENTED_END_SEND] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -107,6 +111,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, [INSTRUMENTED_END_FOR] = END_FOR, + [INSTRUMENTED_END_SEND] = END_SEND, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -142,6 +147,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index d9f4b4adfa48e0..def18489556adc 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -39,6 +39,7 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); @@ -248,6 +249,7 @@ sys_trace_instruction_func( assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -304,6 +306,7 @@ sys_trace_line_func( int line = _PyLong_AsInt(args[1]); assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -334,6 +337,7 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 52822c6a99a46c..842dbbb49d4ba3 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -42,6 +42,10 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1+1; case INSTRUMENTED_END_FOR: return 2; + case END_SEND: + return 2; + case INSTRUMENTED_END_SEND: + return 2; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -432,6 +436,10 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0+0; case INSTRUMENTED_END_FOR: return 0; + case END_SEND: + return 1; + case INSTRUMENTED_END_SEND: + return 1; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -813,6 +821,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [PUSH_NULL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [END_SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_END_SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NEGATIVE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_INVERT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9281ce70d735a4..48825405f39748 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,17 +4,18 @@ static void *opcode_targets[256] = { &&TARGET_PUSH_NULL, &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, + &&TARGET_END_SEND, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_NOP, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_AND_BRANCH_FLOAT, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,38 +55,38 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_COMPARE_AND_BRANCH_FLOAT, &&TARGET_COMPARE_AND_BRANCH_INT, &&TARGET_COMPARE_AND_BRANCH_STR, &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_TUPLE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -142,7 +142,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_COMPARE_AND_BRANCH, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, @@ -251,7 +251,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_END_FOR, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, &&TARGET_DO_TRACING From cdb2bda5648b072713bddacceaed5c168f45f23e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 13:48:11 +0000 Subject: [PATCH 040/116] Correctly set StopIteration in INSTRUMENTED_END_FOR. --- Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5b1ee975cb0944..5449784fc5d498 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,11 +172,11 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, discard --)) { + inst(INSTRUMENTED_END_FOR, (receiver, value --)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetObject(PyExc_StopIteration, value); monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); PyErr_SetRaisedException(NULL); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1c1e469aaa19fd..2c932829014f31 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -236,17 +236,17 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *discard = PEEK(1); + PyObject *value = PEEK(1); PyObject *receiver = PEEK(2); /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetObject(PyExc_StopIteration, value); monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - Py_DECREF(discard); + Py_DECREF(value); STACK_SHRINK(2); DISPATCH(); } From 0148fa35da11156db5d6efb8c8ec2b4521534680 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 15:35:48 +0000 Subject: [PATCH 041/116] Set last traced line when jumping in debugger. --- Objects/frameobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3e5d347ff5dca8..ced90a91d7b738 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -857,6 +857,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ + f->f_last_traced_line = new_lineno; f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; return 0; From c5fb4f48d58eb28b2b244e57160ee487a351bdc9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 13:32:03 +0000 Subject: [PATCH 042/116] Fix test_dis to account for END_SEND. --- Lib/test/test_dis.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b80281580c4083..92ead0e71fec08 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -479,8 +479,7 @@ async def _asyncwith(c): YIELD_VALUE 2 RESUME 3 JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) - >> SWAP 2 - POP_TOP + >> END_SEND POP_TOP %3d LOAD_CONST 1 (1) @@ -492,11 +491,11 @@ async def _asyncwith(c): CALL 2 GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 3 (to 64) + >> SEND 3 (to 62) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 54) - >> POP_TOP + JUMP_BACKWARD_NO_INTERRUPT 5 (to 52) + >> END_SEND POP_TOP %3d LOAD_CONST 2 (2) @@ -504,21 +503,20 @@ async def _asyncwith(c): RETURN_CONST 0 (None) %3d >> CLEANUP_THROW - JUMP_BACKWARD 27 (to 24) + JUMP_BACKWARD 26 (to 24) >> CLEANUP_THROW - JUMP_BACKWARD 9 (to 64) + JUMP_BACKWARD 9 (to 62) >> PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 4 (to 102) + >> SEND 4 (to 100) YIELD_VALUE 3 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 88) >> CLEANUP_THROW - >> SWAP 2 - POP_TOP - POP_JUMP_IF_TRUE 1 (to 110) + >> END_SEND + POP_JUMP_IF_TRUE 1 (to 106) RERAISE 2 >> POP_TOP POP_EXCEPT From 7fa431b603992cfb4917c9e11450d2ce0acb8baf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 13:32:47 +0000 Subject: [PATCH 043/116] Add a few more tests. --- Lib/test/test_monitoring.py | 134 +++++++++++++++++++++++++++++++++--- Python/instrumentation.c | 4 +- 2 files changed, 127 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 37961cc981fb83..faa6f6b9073c81 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -608,29 +608,43 @@ def func2(): class ExceptionRecorder: + event_type = E.RAISE + def __init__(self, events): self.events = events def __call__(self, code, offset, exc): - self.events.append(type(exc)) + self.events.append(("raise", type(exc))) -class ExceptionMontoringTest(unittest.TestCase): +class CheckEvents(unittest.TestCase): - def check_events(self, func, expected, tool=TEST_TOOL, events=E.RAISE): + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: self.assertEqual(sys.monitoring._all_events(), {}) event_list = [] - recorder = ExceptionRecorder(event_list) - sys.monitoring.register_callback(tool, events, recorder) - sys.monitoring.set_events(tool, events) + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_events(tool, all_events) func() sys.monitoring.set_events(tool, 0) - sys.monitoring.register_callback(tool, events, None) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) self.assertEqual(event_list, expected) finally: sys.monitoring.set_events(tool, 0) - sys.monitoring.register_callback(tool, events, None) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + +class StopiterationRecorder(ExceptionRecorder): + + event_type = E.STOP_ITERATION +class ExceptionMontoringTest(CheckEvents): + + recorder = ExceptionRecorder def test_simple_try_except(self): @@ -642,7 +656,7 @@ def func1(): line = 5 line = 6 - self.check_events(func1, [KeyError]) + self.check_events(func1, [("raise", KeyError)]) def gen(): yield 1 @@ -652,4 +666,104 @@ def implicit_stop_iteration(): for _ in gen(): pass - self.check_events(implicit_stop_iteration, [StopIteration], events=E.STOP_ITERATION) + self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + +class LineRecorder: + + event_type = E.LINE + + + def __init__(self, events): + self.events = events + + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) + +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class CEventRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append((self.event_name, func.__name__, arg)) + +class CReturnRecorder(CEventRecorder): + + event_type = E.C_RETURN + event_name = "C return" + +class CRaiseRecorder(CEventRecorder): + + event_type = E.C_RAISE + event_name = "C raise" + +MANY_RECORDERS = ExceptionRecorder, CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestManyEvents(CheckEvents): + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func1', None), + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func2', None), + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func3', None), + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ab254aff29679a..9051eb4e550c60 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1403,8 +1403,10 @@ initialize_lines(PyCodeObject *code) switch (inst.deinstrumented_opcode) { case END_ASYNC_FOR: case END_FOR: + case END_SEND: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * END_SEND cannot start a line, as it is skipped by SEND * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; break; @@ -1807,7 +1809,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, return NULL; } if (_Py_popcount32(event) != 1) { - PyErr_SetString(PyExc_ValueError, "The callaback can only be set for one event at a time"); + PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time"); } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { From 25bbc614ee6575c1d5bc58c05530d5371acd1c17 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 16:00:29 +0000 Subject: [PATCH 044/116] delete commented out code --- Python/legacy_tracing.c | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index def18489556adc..5d9fc6917e5a80 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -169,65 +169,12 @@ sys_trace_return( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { - PyThreadState *tstate = _PyThreadState_GET(); assert(!PyErr_Occurred()); assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); assert(PyCode_Check(args[0])); - PyCodeObject *code = (PyCodeObject *)args[0]; PyObject *val = args[2]; PyObject *res = call_trace_func(self, val); -// if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && -// res != NULL && tstate->c_tracefunc != NULL -// ) { -// /* Fake PEP 380 StopIteration exception event */ -// Py_DECREF(res); -// PyObject *stop_iter_args[2] = { NULL, val }; -// PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, -// &stop_iter_args[1], -// 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, -// NULL); -// if (exc == NULL) { -// return NULL; -// } -// /* The event occurs in the caller frame, for historical reasons */ -// _PyInterpreterFrame *frame = _PyEval_GetFrame(); -// frame = frame->previous; -// while (frame && _PyFrame_IsIncomplete(frame)) { -// frame = frame->previous; -// } -// PyFrameObject* frame_obj = NULL; -// if (frame != NULL) { -// frame_obj = _PyFrame_GetFrameObject(frame); -// } else { -// PyErr_SetString(PyExc_SystemError, -// "Missing frame when calling trace function."); -// return NULL; -// } -// if (frame_obj == NULL) { -// Py_DECREF(exc); -// return NULL; -// } -// PyObject *type = (PyObject *)Py_TYPE(exc); -// PyObject *tb = PyException_GetTraceback(exc); -// if (tb == NULL) { -// tb = Py_NewRef(Py_None); -// } -// PyObject * tuple = PyTuple_Pack(3, type, exc, tb); -// Py_DECREF(tb); -// if (tuple == NULL) { -// Py_DECREF(exc); -// return NULL; -// } -// Py_INCREF(frame_obj); -// int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, tuple); -// Py_DECREF(frame_obj); -// Py_DECREF(tuple); -// Py_DECREF(exc); -// if (err) { -// return NULL; -// } -// } return res; } From 5629a3e859f6e34755957098b6780f67d164896b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 16:01:37 +0000 Subject: [PATCH 045/116] Fix last (known) inconsistency in sys.settrace behaviour --- Lib/test/test_sys_settrace.py | 17 +++++++++++++++++ Objects/frameobject.c | 1 + 2 files changed, 18 insertions(+) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..9cf5d3e9e19b60 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2809,5 +2809,22 @@ def foo(*args): sys.settrace(sys.gettrace()) +class TestLinesAfterTraceStarted(TraceTestCase): + + def test_events(self): + tracer = Tracer() + sys._getframe().f_trace = tracer.trace + sys.settrace(tracer.trace) + line = 4 + line = 5 + sys.settrace(None) + self.compare_events( + TestLinesAfterTraceStarted.test_events.__code__.co_firstlineno, + tracer.events, [ + (4, 'line'), + (5, 'line'), + (6, 'line')]) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ced90a91d7b738..fb8e6423b6e6fe 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -879,6 +879,7 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) v = NULL; } Py_XSETREF(f->f_trace, Py_XNewRef(v)); + f->f_last_traced_line = -1; return 0; } From 477cc53e5709d21e8fb92297dc68043751d33767 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Feb 2023 17:25:07 +0000 Subject: [PATCH 046/116] More clearly differentiate between instrumented events and non-instrumented events. --- Include/cpython/code.h | 10 +- Include/internal/pycore_instruments.h | 1 - Include/internal/pycore_interp.h | 18 +++- Python/ceval.c | 18 +++- Python/instrumentation.c | 141 +++++++++++++++----------- Python/pystate.c | 14 ++- 6 files changed, 127 insertions(+), 75 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ffb6d513fcf7fd..b6d737f7c3fbcb 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -9,10 +9,11 @@ extern "C" { #endif #define PY_MONITORING_EVENTS 16 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 -typedef struct _Py_MonitoringMatrix { - uint8_t tools[PY_MONITORING_EVENTS]; -} _Py_MonitoringMatrix; +typedef struct _Py_InstrumentationMatrix { + uint8_t tools[PY_MONITORING_INSTRUMENTED_EVENTS]; +} _Py_InstrumentationMatrix; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -55,7 +56,8 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_MonitoringMatrix matrix; + _Py_InstrumentationMatrix local_instrumentation; + _Py_InstrumentationMatrix current_instrumentation; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 34dd2551f8aaa6..c7c438ff05ca71 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -27,7 +27,6 @@ extern "C" { #define PY_MONITORING_EVENT_BRANCH 8 #define PY_MONITORING_EVENT_STOP_ITERATION 9 -#define PY_MONITORING_INSTRUMENTED_EVENTS 10 /* Grouped events */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 184db0505864e6..1cf517864a8abc 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -190,14 +190,14 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_MonitoringMatrix monitoring_matrix; - uint8_t required_monitoring_bytes; - /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ - struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; - PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + _Py_InstrumentationMatrix instrumented_events; + uint8_t other_events[PY_MONITORING_EVENTS - PY_MONITORING_INSTRUMENTED_EVENTS]; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; + /* Tools numbered 0-7 */ + struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; Py_ssize_t sys_profiling_threads; Py_ssize_t sys_tracing_threads; @@ -248,6 +248,14 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +static inline uint8_t +_PyInterpreterState_GetTools(PyInterpreterState * is, int event) +{ + assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS && event < PY_MONITORING_EVENTS); + return is->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; +} + + #ifdef __cplusplus } #endif diff --git a/Python/ceval.c b/Python/ceval.c index 3c6b701f120e99..3f329d5df9ca44 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1935,13 +1935,21 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, return 0; } + static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { - if (tstate->interp->monitoring_matrix.tools[event] == 0) { - return; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (frame->f_code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event] == 0) { + return; + } + } + else { + if (_PyInterpreterState_GetTools(tstate->interp, event) == 0) { + return; + } } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); @@ -1961,7 +1969,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_UNWIND) == 0) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); @@ -1973,7 +1981,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_EXCEPTION_HANDLED] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_EXCEPTION_HANDLED) == 0) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -1984,7 +1992,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_THROW) == 0) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9051eb4e550c60..e6886c30a8b373 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -161,9 +161,9 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -171,30 +171,30 @@ matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) return true; } -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +static inline _Py_InstrumentationMatrix +matrix_sub(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + _Py_InstrumentationMatrix res; + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; } -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +static inline _Py_InstrumentationMatrix +matrix_and(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + _Py_InstrumentationMatrix res; + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; } static inline bool -matrix_empty(_Py_MonitoringMatrix m) +matrix_empty(_Py_InstrumentationMatrix m) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -203,10 +203,10 @@ matrix_empty(_Py_MonitoringMatrix m) } static inline int -multiple_tools(_Py_MonitoringMatrix m) +multiple_tools(_Py_InstrumentationMatrix *m) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + if (_Py_popcount32(m->tools[i]) > 1) { return true; } } @@ -214,21 +214,33 @@ multiple_tools(_Py_MonitoringMatrix m) } static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +interpreter_set_tool_event(PyInterpreterState *interp, int event, int tool, int val) { assert(0 <= event && event < PY_MONITORING_EVENTS); assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); + uint8_t *tools; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + tools = &interp->instrumented_events.tools[event]; + } + else { + tools = &interp->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; + } + *tools &= ~(1 << tool); + *tools |= (val << tool); } static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +interpeter_get_events(PyInterpreterState *interp, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { + for (int e = 0; e < PY_MONITORING_INSTRUMENTED_EVENTS; e++) { + if ((interp->instrumented_events.tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + for (int e = PY_MONITORING_INSTRUMENTED_EVENTS; e < PY_MONITORING_EVENTS; e++) { + if ((interp->other_events[e] >> tool_id) & 1) { result |= (1 << e); } } @@ -829,6 +841,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(instruction_has_event(code, offset)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { @@ -839,7 +852,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); @@ -861,7 +874,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -875,6 +888,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data && code->_co_instrumentation.monitoring_data->tools @@ -883,7 +897,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); + assert(PyInterpreterState_Get()->instrumented_events.tools[event] == tools); assert(_Py_popcount32(tools) == 1); } if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { @@ -894,7 +908,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->line_tools ) { @@ -911,7 +925,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->per_instruction_tools ) { @@ -939,7 +953,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_per_instruction(code, offset); @@ -999,14 +1013,18 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring && monitoring->tools) { + assert(monitoring != NULL); + if (event >= PY_MONITORING_INSTRUMENTED_EVENTS) { + return _PyInterpreterState_GetTools(PyInterpreterState_Get(), event); + } + else if (monitoring->tools) { tools = monitoring->tools[i]; } else { - tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; } - CHECK((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - CHECK((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + CHECK((tools & PyInterpreterState_Get()->instrumented_events.tools[event]) == tools); + CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); return tools; } @@ -1053,7 +1071,7 @@ _Py_call_instrumentation( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1073,7 +1091,7 @@ _Py_call_instrumentation_arg( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1093,7 +1111,7 @@ _Py_call_instrumentation_2args( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1153,9 +1171,9 @@ _Py_call_instrumentation_exc_vector( assert(args[1] == NULL); args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS); + uint8_t tools = tstate->interp->instrumented_events.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1221,7 +1239,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; @@ -1234,7 +1252,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[i] : - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -1274,7 +1292,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; assert(instrumentation_data->per_instruction_opcodes); @@ -1285,7 +1303,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1359,7 +1377,8 @@ initialize_tools(PyCodeObject *code) assert(event > 0); } assert(event >= 0); - tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); + tools[i] = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; CHECK(tools[i] != 0); } else { @@ -1444,7 +1463,7 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { - line_tools[i] = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; } } @@ -1458,14 +1477,14 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->matrix = (_Py_MonitoringMatrix){ 0 }; + code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_InstrumentationMatrix){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - bool multitools = multiple_tools(interp->monitoring_matrix); + bool multitools = multiple_tools(&interp->instrumented_events); if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { @@ -1474,7 +1493,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_tools(code); } - if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { + if (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1492,7 +1511,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } - if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { @@ -1539,7 +1558,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (is_instrumentation_up_to_date(code, interp)) { assert(interp->monitoring_version == 0 || - matrix_equals(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix) + matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events) ); return 0; } @@ -1547,20 +1566,20 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_MonitoringMatrix new_events; - _Py_MonitoringMatrix removed_events; + _Py_InstrumentationMatrix new_events; + _Py_InstrumentationMatrix removed_events; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_data->matrix; - new_events = interp->monitoring_matrix; + removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; + new_events = interp->instrumented_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix); - new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_data->matrix); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events); + new_events = matrix_sub(interp->instrumented_events, code->_co_instrumentation.monitoring_data->current_instrumentation); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; + code->_co_instrumentation.monitoring_data->current_instrumentation = interp->instrumented_events; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { //sanity_check_instrumentation(code); @@ -1681,13 +1700,13 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { PyInterpreterState *interp = _PyInterpreterState_Get(); assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); - uint32_t existing_events = matrix_get_events(&interp->monitoring_matrix, tool_id); + uint32_t existing_events = interpeter_get_events(interp, tool_id); if (existing_events == events) { return; } for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; - matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); + interpreter_set_tool_event(interp, e, tool_id, val); } interp->monitoring_version++; @@ -1842,7 +1861,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = matrix_get_events(&interp->monitoring_matrix, tool_id); + _PyMonitoringEventSet event_set = interpeter_get_events(interp, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1942,7 +1961,13 @@ monitoring__all_events_impl(PyObject *module) return NULL; } for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools = interp->monitoring_matrix.tools[e]; + uint8_t tools; + if (e < PY_MONITORING_INSTRUMENTED_EVENTS) { + tools = interp->instrumented_events.tools[e]; + } + else { + tools = interp->other_events[e]; + } if (tools == 0) { continue; } diff --git a/Python/pystate.c b/Python/pystate.c index 84ae45874dbb97..aebe4ae9e1e267 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -641,8 +641,13 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); + for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->instrumented_events.tools[i] = 0; + } + for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->other_events[i] = 0; + } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; } @@ -773,8 +778,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); + for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->instrumented_events.tools[i] = 0; + } + for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->other_events[i] = 0; + } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->tools[t].instrument_callables[i]); } From 23b5f5ed14684fb4d8315a5817dfd42d4fe582a4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 21 Feb 2023 14:48:18 +0000 Subject: [PATCH 047/116] Add support for sys.monitoring.MISSING --- Include/internal/pycore_instruments.h | 6 ++---- Lib/test/test_monitoring.py | 3 +-- Python/bytecodes.c | 6 ++++-- Python/instrumentation.c | 12 +++++++++++- Python/legacy_tracing.c | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index c7c438ff05ca71..91fba602827b03 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -43,10 +43,6 @@ extern "C" { /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ -/* Temporary and internal events */ - -// #define PY_INSTRUMENT_PEP_523 50 -/* #define PY_INSTRUMENT_JIT_API 17 -- Reserved */ typedef uint32_t _PyMonitoringEventSet; @@ -99,6 +95,8 @@ _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); +extern PyObject _PyInstrumentation_MISSING; + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index faa6f6b9073c81..3e9a27d2984157 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -46,10 +46,9 @@ def test_has_objects(self): # m.get_local_events # m.set_local_events m.register_callback - # m.insert_marker - # m.remove_marker m.restart_events m.DISABLE + m.MISSING def test_tool(self): sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5449784fc5d498..4f8761dff8e035 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2507,7 +2507,8 @@ dummy_func( int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, function, arg); @@ -2605,7 +2606,8 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e6886c30a8b373..ce3d89ee2901bc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -4,6 +4,7 @@ #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_namespace.h" +#include "pycore_object.h" #include "pycore_opcode.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" @@ -13,7 +14,13 @@ static PyObject DISABLE = { - 1 << 30, + _PyObject_IMMORTAL_REFCNT, + &PyBaseObject_Type +}; + +PyObject _PyInstrumentation_MISSING = +{ + _PyObject_IMMORTAL_REFCNT, &PyBaseObject_Type }; @@ -2013,6 +2020,9 @@ PyObject *_Py_CreateMonitoringObject(void) if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) { goto error; } + if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) { + goto error; + } PyObject *events = _PyNamespace_New(NULL); if (events == NULL) { goto error; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 5d9fc6917e5a80..348138e3649d19 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -91,7 +91,7 @@ sys_profile_call_or_return( * convert to builtin method */ /* If no arg, skip */ - if (self_arg == Py_None) { + if (self_arg == &_PyInstrumentation_MISSING) { Py_RETURN_NONE; } PyObject *meth = Py_TYPE(callable)->tp_descr_get( From dfc18c53610af938c797be7adef18a6df4ba11b4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 10:23:22 +0000 Subject: [PATCH 048/116] Add support for local (per-code-object) event monitoring. --- Include/cpython/code.h | 12 +- Include/internal/pycore_instruments.h | 18 +- Include/internal/pycore_interp.h | 10 +- Lib/test/test_monitoring.py | 78 +++++- Python/bytecodes.c | 14 +- Python/ceval.c | 73 ++++-- Python/clinic/instrumentation.c.h | 73 +++++- Python/generated_cases.c.h | 20 +- Python/instrumentation.c | 362 ++++++++++++++++++-------- Python/pystate.c | 10 +- 10 files changed, 485 insertions(+), 185 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b6d737f7c3fbcb..f2e31087e58d66 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -9,11 +9,11 @@ extern "C" { #endif #define PY_MONITORING_EVENTS 16 -#define PY_MONITORING_INSTRUMENTED_EVENTS 10 +#define PY_MONITORING_UNGROUPED_EVENTS 14 -typedef struct _Py_InstrumentationMatrix { - uint8_t tools[PY_MONITORING_INSTRUMENTED_EVENTS]; -} _Py_InstrumentationMatrix; +typedef struct _Py_Monitors { + uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; +} _Py_Monitors; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -56,8 +56,8 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_InstrumentationMatrix local_instrumentation; - _Py_InstrumentationMatrix current_instrumentation; + _Py_Monitors local_instrumentation; + _Py_Monitors current_instrumentation; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 91fba602827b03..f34bd074e6a92f 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -27,18 +27,20 @@ extern "C" { #define PY_MONITORING_EVENT_BRANCH 8 #define PY_MONITORING_EVENT_STOP_ITERATION 9 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 -/* Grouped events */ +/* Exceptional events */ -#define PY_MONITORING_EVENT_C_RETURN 10 -#define PY_MONITORING_EVENT_C_RAISE 11 +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 -/* Exceptional events */ -#define PY_MONITORING_EVENT_RAISE 12 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 -#define PY_MONITORING_EVENT_PY_UNWIND 14 -#define PY_MONITORING_EVENT_PY_THROW 15 +/* Grouped events */ + +#define PY_MONITORING_EVENT_C_RETURN 14 +#define PY_MONITORING_EVENT_C_RAISE 15 /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1cf517864a8abc..964e4f5e908ff9 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -190,8 +190,7 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_InstrumentationMatrix instrumented_events; - uint8_t other_events[PY_MONITORING_EVENTS - PY_MONITORING_INSTRUMENTED_EVENTS]; + _Py_Monitors instrumented_events; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; @@ -248,13 +247,6 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); -static inline uint8_t -_PyInterpreterState_GetTools(PyInterpreterState * is, int event) -{ - assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS && event < PY_MONITORING_EVENTS); - return is->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; -} - #ifdef __cplusplus } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 3e9a27d2984157..9278a474ed66fe 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -43,8 +43,8 @@ def test_has_objects(self): m.get_tool m.get_events m.set_events - # m.get_local_events - # m.set_local_events + m.get_local_events + m.set_local_events m.register_callback m.restart_events m.DISABLE @@ -719,7 +719,7 @@ def func1(): self.check_events(func1, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func1', None), + ('call', 'func1', sys.monitoring.MISSING), ('line', 'func1', 1), ('line', 'func1', 2), ('line', 'func1', 3), @@ -735,7 +735,7 @@ def func2(): self.check_events(func2, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func2', None), + ('call', 'func2', sys.monitoring.MISSING), ('line', 'func2', 1), ('line', 'func2', 2), ('call', 'append', [2]), @@ -756,7 +756,7 @@ def func3(): self.check_events(func3, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func3', None), + ('call', 'func3', sys.monitoring.MISSING), ('line', 'func3', 1), ('line', 'func3', 2), ('line', 'func3', 3), @@ -766,3 +766,71 @@ def func3(): ('line', 'func3', 6), ('line', 'check_events', 11), ('call', 'set_events', 2)]) + +class TestLocalEvents(unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(func.__code__, tool, all_events) + func() + sys.monitoring.set_local_events(func.__code__, tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_local_events(func.__code__, tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6)]) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4f8761dff8e035..6d1d8cd7f4495b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -177,7 +177,9 @@ dummy_func( * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } DECREF_INPUTS(); @@ -190,7 +192,9 @@ dummy_func( inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -794,7 +798,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2215,7 +2219,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2244,7 +2248,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ diff --git a/Python/ceval.c b/Python/ceval.c index 3f329d5df9ca44..0c1276e13c21ef 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -178,7 +178,10 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event); + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -899,7 +902,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); exception_unwind: { @@ -1935,22 +1938,12 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, return 0; } - -static void -monitor_raise(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event) -{ - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - if (frame->f_code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event] == 0) { - return; - } - } - else { - if (_PyInterpreterState_GetTools(tstate->interp, event) == 0) { - return; - } - } +static int +do_monitor_exc( + PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event +) { + assert(event < PY_MONITORING_UNGROUPED_EVENTS); PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err; @@ -1961,15 +1954,53 @@ monitor_raise(PyThreadState *tstate, else { Py_DECREF(exc); } + return err; +} + +static inline int +no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + _PyCoInstrumentationData *data = frame->f_code->_co_instrumentation.monitoring_data; + if (data) { + if (data->current_instrumentation.tools[event] == 0) { + return 1; + } + } + else { + if (tstate->interp->instrumented_events.tools[event] == 0) { + return 1; + } + } + return 0; } +static void +monitor_raise( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); +} + +static int +monitor_stop_iteration( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; + } + return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); +} static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_UNWIND) == 0) { + + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); @@ -1981,7 +2012,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_EXCEPTION_HANDLED) == 0) { + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -1992,7 +2023,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_THROW) == 0) { + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index feecf7695a2a2a..57021191e29df6 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -194,6 +194,77 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(monitoring_get_local_events__doc__, +"get_local_events($module, code, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_LOCAL_EVENTS_METHODDEF \ + {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, + +static PyObject * +monitoring_get_local_events_impl(PyObject *module, PyObject *code, + int tool_id); + +static PyObject * +monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + int tool_id; + + if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { + goto exit; + } + code = args[0]; + tool_id = _PyLong_AsInt(args[1]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_local_events_impl(module, code, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_local_events__doc__, +"set_local_events($module, code, tool_id, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_LOCAL_EVENTS_METHODDEF \ + {"set_local_events", _PyCFunction_CAST(monitoring_set_local_events), METH_FASTCALL, monitoring_set_local_events__doc__}, + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, PyObject *code, + int tool_id, int event_set); + +static PyObject * +monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + int tool_id; + int event_set; + + if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + tool_id = _PyLong_AsInt(args[1]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event_set = _PyLong_AsInt(args[2]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_local_events_impl(module, code, tool_id, event_set); + +exit: + return return_value; +} + PyDoc_STRVAR(monitoring_restart_events__doc__, "restart_events($module, /)\n" "--\n" @@ -227,4 +298,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=d4c412a002392e2b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9e5f270f3ce1e945 input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2c932829014f31..74b69f697ccc00 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -242,7 +242,9 @@ * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -265,7 +267,9 @@ PyObject *receiver = PEEK(2); if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -1010,7 +1014,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2793,7 +2797,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2826,7 +2830,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3160,7 +3164,8 @@ int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, function, arg); @@ -3234,7 +3239,8 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ce3d89ee2901bc..2619716fef95f8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -168,9 +168,9 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +matrix_equals(_Py_Monitors a, _Py_Monitors b) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -178,30 +178,40 @@ matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) return true; } -static inline _Py_InstrumentationMatrix -matrix_sub(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +static inline _Py_Monitors +matrix_sub(_Py_Monitors a, _Py_Monitors b) { - _Py_InstrumentationMatrix res; - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; } -static inline _Py_InstrumentationMatrix -matrix_and(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +static inline _Py_Monitors +matrix_and(_Py_Monitors a, _Py_Monitors b) { - _Py_InstrumentationMatrix res; - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; } +static inline _Py_Monitors +matrix_or(_Py_Monitors a, _Py_Monitors b) +{ + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + res.tools[i] = a.tools[i] | b.tools[i]; + } + return res; +} + static inline bool -matrix_empty(_Py_InstrumentationMatrix m) +matrix_empty(_Py_Monitors m) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -210,9 +220,9 @@ matrix_empty(_Py_InstrumentationMatrix m) } static inline int -multiple_tools(_Py_InstrumentationMatrix *m) +multiple_tools(_Py_Monitors *m) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -220,34 +230,12 @@ multiple_tools(_Py_InstrumentationMatrix *m) return false; } -static inline void -interpreter_set_tool_event(PyInterpreterState *interp, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - uint8_t *tools; - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - tools = &interp->instrumented_events.tools[event]; - } - else { - tools = &interp->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; - } - *tools &= ~(1 << tool); - *tools |= (val << tool); -} - static inline _PyMonitoringEventSet -interpeter_get_events(PyInterpreterState *interp, int tool_id) +get_events(_Py_Monitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_INSTRUMENTED_EVENTS; e++) { - if ((interp->instrumented_events.tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - for (int e = PY_MONITORING_INSTRUMENTED_EVENTS; e < PY_MONITORING_EVENTS; e++) { - if ((interp->other_events[e] >> tool_id) & 1) { + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } } @@ -853,7 +841,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; - if (monitoring->tools[offset] == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (monitoring->tools[offset] == 0) { de_instrument(code, offset, event); } } @@ -861,12 +849,21 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) /* Single tool */ uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (((single_tool & tools) == single_tool)) { de_instrument(code, offset, event); } } } +#ifndef NDEBUG +bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) +{ + int global_tools = PyInterpreterState_Get()->instrumented_events.tools[event]; + int local_tools = code->_co_instrumentation.monitoring_data->local_instrumentation.tools[event]; + return tools == ((global_tools | local_tools) & tools); +} +#endif + static void remove_line_tools(PyCodeObject * code, int offset, int tools) { @@ -889,7 +886,6 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } } - static void add_tools(PyCodeObject * code, int offset, int event, int tools) { @@ -904,18 +900,17 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - assert(PyInterpreterState_Get()->instrumented_events.tools[event] == tools); + assert(_Py_popcount32(tools) == 1); + assert(tools_is_subset_for_event(code, event, tools)); assert(_Py_popcount32(tools) == 1); } - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - instrument(code, offset); - } + instrument(code, offset); } static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->line_tools ) { @@ -932,7 +927,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->per_instruction_tools ) { @@ -1013,24 +1008,44 @@ static inline int most_significant_bit(uint8_t bits) { } } +static bool +is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + +#ifndef NDEBUG +static bool +instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) +{ + _Py_Monitors expected = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + return matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, + expected); +} +#endif + static inline uint8_t get_tools_for_instruction(PyCodeObject * code, int i, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - assert(monitoring != NULL); - if (event >= PY_MONITORING_INSTRUMENTED_EVENTS) { - return _PyInterpreterState_GetTools(PyInterpreterState_Get(), event); - } - else if (monitoring->tools) { + if (monitoring->tools) { tools = monitoring->tools[i]; } else { + if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + assert(event == PY_MONITORING_EVENT_C_RAISE || + event == PY_MONITORING_EVENT_C_RETURN); + event = PY_MONITORING_EVENT_CALL; + } tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; } - CHECK((tools & PyInterpreterState_Get()->instrumented_events.tools[event]) == tools); + CHECK(tools_is_subset_for_event(code, event, tools)); CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); return tools; } @@ -1065,20 +1080,14 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -static bool -is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) -{ - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; -} - int _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1097,8 +1106,8 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1117,8 +1126,8 @@ _Py_call_instrumentation_2args( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1179,8 +1188,7 @@ _Py_call_instrumentation_exc_vector( args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); int offset = instr - _PyCode_CODE(code); - assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS); - uint8_t tools = tstate->interp->instrumented_events.tools[event]; + uint8_t tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1245,8 +1253,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, { frame->prev_instr = instr; PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; @@ -1259,7 +1267,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[i] : - interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; + (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE] | + code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_LINE] + ); PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -1298,8 +1308,8 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; assert(instrumentation_data->per_instruction_opcodes); @@ -1310,8 +1320,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - + (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] | + code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION] + ); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { return -1; @@ -1464,34 +1475,48 @@ initialize_lines(PyCodeObject *code) } static void -initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) +initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) { uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { - line_tools[i] = interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE]; } } -int -update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +static +int allocate_instrumentation_data(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); + if (code->_co_instrumentation.monitoring_data == NULL) { code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); if (code->_co_instrumentation.monitoring_data == NULL) { PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_InstrumentationMatrix){ 0 }; + code->_co_instrumentation.monitoring_data->local_instrumentation = (_Py_Monitors){ 0 }; + code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_Monitors){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - bool multitools = multiple_tools(&interp->instrumented_events); + return 0; +} + +static int +update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors all_events = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + bool multitools = multiple_tools(&all_events); if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { @@ -1500,7 +1525,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_tools(code); } - if (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]) { + if (all_events.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1515,10 +1540,10 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - initialize_line_tools(code, interp); + initialize_line_tools(code, &all_events); } } - if (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { @@ -1563,9 +1588,10 @@ int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { - if (is_instrumentation_up_to_date(code, interp)) { - assert(interp->monitoring_version == 0 || - matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events) + if (is_version_up_to_date(code, interp)) { + assert( + interp->monitoring_version == 0 || + instrumentation_cross_checks(interp, code) ); return 0; } @@ -1573,23 +1599,28 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_InstrumentationMatrix new_events; - _Py_InstrumentationMatrix removed_events; + _Py_Monitors active_events = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors new_events; + _Py_Monitors removed_events; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; - new_events = interp->instrumented_events; + new_events = active_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events); - new_events = matrix_sub(interp->instrumented_events, code->_co_instrumentation.monitoring_data->current_instrumentation); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, active_events); + new_events = matrix_sub(active_events, code->_co_instrumentation.monitoring_data->current_instrumentation); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->current_instrumentation = interp->instrumented_events; + code->_co_instrumentation.monitoring_data->current_instrumentation = active_events; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { - //sanity_check_instrumentation(code); +#ifdef INSTRUMENT_DEBUG + sanity_check_instrumentation(code); +#endif return 0; } /* Insert instrumentation */ @@ -1702,24 +1733,56 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { } } +static void +set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t *tools = &m->tools[e]; + int val = (events >> e) & 1; + *tools &= ~(1 << tool_id); + *tools |= (val << tool_id); + } +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); - assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); - uint32_t existing_events = interpeter_get_events(interp, tool_id); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + uint32_t existing_events = get_events(&interp->instrumented_events, tool_id); if (existing_events == events) { return; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - int val = (events >> e) & 1; - interpreter_set_tool_event(interp, e, tool_id, val); - } + set_events(&interp->instrumented_events, tool_id, events); interp->monitoring_version++; - instrument_all_executing_code_objects(interp); } +int +_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_Get(); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors *local = &code->_co_instrumentation.monitoring_data->local_instrumentation; + uint32_t existing_events = get_events(local, tool_id); + if (existing_events == events) { + return 0; + } + set_events(local, tool_id, events); + if (is_version_up_to_date(code, interp)) { + /* Force instrumentation update */ + code->_co_instrumentation.monitoring_version = UINT64_MAX; + } + _Py_Instrument(code, interp); + return 0; +} + /*[clinic input] module monitoring [clinic start generated code]*/ @@ -1867,8 +1930,8 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = interpeter_get_events(interp, tool_id); + _Py_Monitors *m = &_PyInterpreterState_Get()->instrumented_events; + _PyMonitoringEventSet event_set = get_events(m, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1896,13 +1959,86 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); return NULL; } - if (event_set & (1 << PY_MONITORING_EVENT_CALL)) { - event_set |= C_RETURN_EVENTS; - } + event_set &= ~C_RETURN_EVENTS; _PyMonitoring_SetEvents(tool_id, event_set); Py_RETURN_NONE; } +/*[clinic input] +monitoring.get_local_events + + code: object + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_local_events_impl(PyObject *module, PyObject *code, + int tool_id) +/*[clinic end generated code: output=22fad50d2e6404a7 input=c14650d27de48264]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return NULL; + } + if (check_valid_tool(tool_id)) { + return NULL; + } + _PyMonitoringEventSet event_set = 0; + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + _Py_Monitors *m = &((PyCodeObject *)code)-> + _co_instrumentation.monitoring_data->local_instrumentation; + if ((m->tools[e] >> tool_id) & 1) { + event_set |= (1 << e); + } + } + return PyLong_FromUnsignedLong(event_set); +} + +/*[clinic input] +monitoring.set_local_events + + code: object + tool_id: int + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, PyObject *code, + int tool_id, int event_set) +/*[clinic end generated code: output=65d616f95cbb76d8 input=2706fbfe062404bf]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return NULL; + } + if (check_valid_tool(tool_id)) { + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + event_set &= ~C_RETURN_EVENTS; + if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] monitoring.restart_events @@ -1967,14 +2103,8 @@ monitoring__all_events_impl(PyObject *module) if (res == NULL) { return NULL; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools; - if (e < PY_MONITORING_INSTRUMENTED_EVENTS) { - tools = interp->instrumented_events.tools[e]; - } - else { - tools = interp->other_events[e]; - } + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t tools = interp->instrumented_events.tools[e]; if (tools == 0) { continue; } @@ -1994,6 +2124,8 @@ static PyMethodDef methods[] = { MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF MONITORING_SET_EVENTS_METHODDEF + MONITORING_GET_LOCAL_EVENTS_METHODDEF + MONITORING_SET_LOCAL_EVENTS_METHODDEF MONITORING_RESTART_EVENTS_METHODDEF MONITORING__ALL_EVENTS_METHODDEF {NULL, NULL} // sentinel diff --git a/Python/pystate.c b/Python/pystate.c index aebe4ae9e1e267..2edba326886331 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -641,12 +641,9 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->instrumented_events.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { - interp->other_events[i] = 0; - } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; @@ -778,12 +775,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->instrumented_events.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { - interp->other_events[i] = 0; - } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->tools[t].instrument_callables[i]); From cfb17ed861b18892d5ec574992a7f1a5a3e0444c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 11:14:35 +0000 Subject: [PATCH 049/116] Make sure that tracing/profiling thread counts are correct. --- Include/internal/pycore_interp.h | 4 ++-- Python/pystate.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 5c4b710de08efa..0037fbf0e5a96d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -169,8 +169,8 @@ struct _is { /* Tools numbered 0-7 */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; - Py_ssize_t sys_profiling_threads; - Py_ssize_t sys_tracing_threads; + Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ + Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Python/pystate.c b/Python/pystate.c index 8daece9f4569b1..67c479272631d9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1380,8 +1380,14 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } - tstate->c_profilefunc = NULL; - tstate->c_tracefunc = NULL; + if (tstate->c_profilefunc != NULL) { + tstate->interp->sys_profiling_threads--; + tstate->c_profilefunc = NULL; + } + if (tstate->c_tracefunc != NULL) { + tstate->interp->sys_tracing_threads--; + tstate->c_tracefunc = NULL; + } Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); From c535f76636be9860be55fad3e882c268f3749077 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 11:43:21 +0000 Subject: [PATCH 050/116] Remove linearray field --- Include/cpython/code.h | 2 -- Include/internal/pycore_code.h | 26 --------------- Objects/codeobject.c | 60 ---------------------------------- Python/instrumentation.c | 2 +- Tools/build/deepfreeze.py | 2 -- 5 files changed, 1 insertion(+), 91 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 5e62233e906f5e..b2e32f15d5ca3e 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -115,7 +115,6 @@ typedef struct { PyObject *co_exceptiontable; /* Byte string encoding exception handling \ table */ \ int co_flags; /* CO_..., see below */ \ - short _co_linearray_entry_size; /* Size of each entry in _co_linearray */ \ \ /* The rest are not so impactful on performance. */ \ int co_argcount; /* #arguments, except *args */ \ @@ -144,7 +143,6 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 70533e47be4dfb..6fa79a121e2b7b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -444,32 +444,6 @@ adaptive_counter_backoff(uint16_t counter) { /* Line array cache for tracing */ -extern int _PyCode_CreateLineArray(PyCodeObject *co); - -static inline int -_PyCode_InitLineArray(PyCodeObject *co) -{ - if (co->_co_linearray) { - return 0; - } - return _PyCode_CreateLineArray(co); -} - -static inline int -_PyCode_LineNumberFromArray(PyCodeObject *co, int index) -{ - assert(co->_co_linearray != NULL); - assert(index >= 0); - assert(index < Py_SIZE(co)); - if (co->_co_linearray_entry_size == 2) { - return ((int16_t *)co->_co_linearray)[index]; - } - else { - assert(co->_co_linearray_entry_size == 4); - return ((int32_t *)co->_co_linearray)[index]; - } -} - typedef struct _PyShimCodeDef { const uint8_t *code; int codelen; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1774519cdbbf1f..367cf3095fee93 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -409,8 +409,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_extra = NULL; co->_co_cached = NULL; - co->_co_linearray_entry_size = 0; - co->_co_linearray = NULL; memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); int entry_point = 0; @@ -788,54 +786,6 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -/* Use co_linetable to compute the line number from a bytecode index, addrq. See - lnotab_notes.txt for the details of the lnotab representation. -*/ - -int -_PyCode_CreateLineArray(PyCodeObject *co) -{ - assert(co->_co_linearray == NULL); - PyCodeAddressRange bounds; - int size; - int max_line = 0; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - if (bounds.ar_line > max_line) { - max_line = bounds.ar_line; - } - } - if (max_line < (1 << 15)) { - size = 2; - } - else { - size = 4; - } - co->_co_linearray = PyMem_Malloc(Py_SIZE(co)*size); - if (co->_co_linearray == NULL) { - PyErr_NoMemory(); - return -1; - } - co->_co_linearray_entry_size = size; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - int start = bounds.ar_start / sizeof(_Py_CODEUNIT); - int end = bounds.ar_end / sizeof(_Py_CODEUNIT); - for (int index = start; index < end; index++) { - assert(index < (int)Py_SIZE(co)); - if (size == 2) { - assert(((int16_t)bounds.ar_line) == bounds.ar_line); - ((int16_t *)co->_co_linearray)[index] = bounds.ar_line; - } - else { - assert(size == 4); - ((int32_t *)co->_co_linearray)[index] = bounds.ar_line; - } - } - } - return 0; -} - int PyCode_Addr2Line(PyCodeObject *co, int addrq) { @@ -843,9 +793,6 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) return co->co_firstlineno; } assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); - if (co->_co_linearray) { - return _PyCode_LineNumberFromArray(co, addrq / sizeof(_Py_CODEUNIT)); - } PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -1722,9 +1669,6 @@ code_dealloc(PyCodeObject *co) PyMem_Free(data->per_instruction_tools); } } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - } PyObject_Free(co); } @@ -2307,10 +2251,6 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - co->_co_linearray = NULL; - } } int diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f02e33faf55b25..747f699e5df308 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -282,7 +282,7 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } else { assert(line_delta == -127); - /* Compute from table */ + /* Look it up */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } } diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 511b26a5ce3dc7..bfdaf8ac41c451 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -255,7 +255,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_names = {co_names},") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") - self.write("._co_linearray_entry_size = 0,") self.field(code, "co_argcount") self.field(code, "co_posonlyargcount") self.field(code, "co_kwonlyargcount") @@ -276,7 +275,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") self.write(f"._co_cached = NULL,") - self.write("._co_linearray = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") for i, op in enumerate(code.co_code[::2]): if op == RESUME: From d579d2e73eb77cbd1bd676604f50151663ac73f5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 12:16:00 +0000 Subject: [PATCH 051/116] Rename some fields, shortening names. --- Include/cpython/code.h | 14 +- Include/internal/pycore_interp.h | 11 +- Objects/codeobject.c | 6 +- Python/bytecodes.c | 4 +- Python/ceval.c | 6 +- Python/generated_cases.c.h | 4 +- Python/instrumentation.c | 306 +++++++++++++++---------------- Python/pystate.c | 16 +- 8 files changed, 179 insertions(+), 188 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b2e32f15d5ca3e..ebc832e79974bc 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -70,19 +70,14 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_Monitors local_instrumentation; - _Py_Monitors current_instrumentation; + _Py_Monitors local_monitors; + _Py_Monitors active_monitors; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; uint8_t *per_instruction_opcodes; uint8_t *per_instruction_tools; -} _PyCoInstrumentationData; - -typedef struct { - uint64_t monitoring_version; /* current instrumentation version */ - _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ -} _PyCoInstrumentation; +} _PyCoMonitoringData; // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: @@ -141,7 +136,8 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ - _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ + uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0037fbf0e5a96d..36198d2c41bd33 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -49,10 +49,6 @@ struct _Py_long_state { int max_str_digits; }; -struct _instrumentation_tool { - PyObject *instrument_callables[PY_MONITORING_EVENTS]; -}; - /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -162,15 +158,14 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_Monitors instrumented_events; + _Py_Monitors monitors; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; - /* Tools numbered 0-7 */ - struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; - PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 367cf3095fee93..44cb31a1fcb689 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -402,8 +402,8 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) if (_Py_next_func_version != 0) { _Py_next_func_version++; } - co->_co_instrumentation.monitoring_data = NULL; - co->_co_instrumentation.monitoring_version = 0; + co->_co_monitoring = NULL; + co->_co_instrumentation_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; @@ -1651,7 +1651,7 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } - _PyCoInstrumentationData *data = co->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = co->_co_monitoring; if (data) { if (data->tools) { PyMem_Free(data->tools); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3ee972cbb395d0..df4686fa033a59 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -91,7 +91,7 @@ dummy_func( inst(INSTRUMENTED_RESUME, (--)) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -118,7 +118,7 @@ dummy_func( assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); next_instr--; diff --git a/Python/ceval.c b/Python/ceval.c index e1eb73935fedc7..8c0a80487985fc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1961,14 +1961,14 @@ do_monitor_exc( static inline int no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - _PyCoInstrumentationData *data = frame->f_code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = frame->f_code->_co_monitoring; if (data) { - if (data->current_instrumentation.tools[event] == 0) { + if (data->active_monitors.tools[event] == 0) { return 1; } } else { - if (tstate->interp->instrumented_events.tools[event] == 0) { + if (tstate->interp->monitors.tools[event] == 0) { return 1; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5c125bd824c30e..64cc0aaa581c71 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9,7 +9,7 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -37,7 +37,7 @@ assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; next_instr--; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 747f699e5df308..c0f4d4920cda82 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -168,7 +168,7 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_Monitors a, _Py_Monitors b) +monitors_equals(_Py_Monitors a, _Py_Monitors b) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { @@ -179,7 +179,7 @@ matrix_equals(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_sub(_Py_Monitors a, _Py_Monitors b) +monitors_sub(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -189,7 +189,7 @@ matrix_sub(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_and(_Py_Monitors a, _Py_Monitors b) +monitors_and(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -199,7 +199,7 @@ matrix_and(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_or(_Py_Monitors a, _Py_Monitors b) +monitors_or(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -209,7 +209,7 @@ matrix_or(_Py_Monitors a, _Py_Monitors b) } static inline bool -matrix_empty(_Py_Monitors m) +monitors_empty(_Py_Monitors m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { @@ -297,10 +297,10 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -318,10 +318,10 @@ uint8_t deinstrument(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -346,12 +346,12 @@ Instruction read_instruction(PyCodeObject *code, int offset) Instruction result = (Instruction){0}; int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; result.is_prefix_instrumented = true; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; result.is_prefix_instrumented = true; } while (opcode == EXTENDED_ARG) { @@ -420,7 +420,7 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } static void -dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentationData *data, int i, FILE*out) +dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out) { if (data->per_instruction_opcodes == NULL) { fprintf(out, ", per-inst opcode = NULL"); @@ -436,14 +436,12 @@ dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentati } } - - static void -dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) +dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) { - fprintf(out, "%s matrix:\n", prefix); - for (int event = 0; event < PY_MONITORING_EVENTS; event++) { - fprintf(out, " Event %d: Tools %x\n", event, matrix.tools[event]); + fprintf(out, "%s monitors:\n", prefix); + for (int event = 0; event < PY_MONITORING_UNGROUPED_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -458,16 +456,16 @@ int get_base_opcode_best_attempt(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] == 0) { + if (code->_co_monitoring->per_instruction_opcodes[offset] == 0) { return opcode; } - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - if (code->_co_instrumentation.monitoring_data->lines[offset].original_opcode == 0) { + if (code->_co_monitoring->lines[offset].original_opcode == 0) { return opcode; } - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -483,14 +481,15 @@ int get_base_opcode_best_attempt(PyCodeObject *code, int offset) static void dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = code->_co_monitoring; fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; } - dump_matrix("Global", PyInterpreterState_Get()->monitoring_matrix, out); - dump_matrix("Code", data->matrix, out); + dump_monitors("Global", PyInterpreterState_Get()->monitors, out); + dump_monitors("Code", data->local_monitors, out); + dump_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; for (int i = 0; i < code_len; i++) { @@ -542,14 +541,18 @@ bool valid_opcode(int opcode) { static void sanity_check_instrumentation(PyCodeObject *code) { - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = code->_co_monitoring; if (data == NULL) { return; } - _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; - assert(matrix_equals( - code->_co_instrumentation.monitoring_data->matrix, - tools_matrix) + _Py_Monitors active_monitors = PyInterpreterState_Get()->monitors; + if (code->_co_monitoring) { + _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; + active_monitors = monitors_or(active_monitors, local_monitors); + } + assert(monitors_equals( + code->_co_monitoring->active_monitors, + active_monitors) ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len;) { @@ -584,7 +587,7 @@ sanity_check_instrumentation(PyCodeObject *code) /* RESUME fixup */ event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } - CHECK(tools_matrix.tools[event] != 0); + CHECK(active_monitors.tools[event] != 0); CHECK(inst.deinstrumented_opcode != END_FOR); } if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { @@ -612,8 +615,7 @@ sanity_check_instrumentation(PyCodeObject *code) /* RESUME fixup */ event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } - uint8_t global_tools = tools_matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); + CHECK((active_monitors.tools[event] & local_tools) == local_tools); } else { CHECK(local_tools == 0xff); @@ -641,18 +643,18 @@ de_instrument(PyCodeObject *code, int offset, int event) } switch(opcode) { case INSTRUMENTED_INSTRUCTION: - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; if (opcode != INSTRUMENTED_LINE) { int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = deinstrumented; + code->_co_monitoring->per_instruction_opcodes[offset] = deinstrumented; } break; } /* Intentional fall through */ case INSTRUMENTED_LINE: { - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; int orignal_opcode = lines->original_opcode; int deinstrumented = DE_INSTRUMENT[orignal_opcode]; if (deinstrumented) { @@ -682,12 +684,12 @@ de_instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i]; + opcode = code->_co_monitoring->per_instruction_opcodes[i]; } if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { @@ -702,7 +704,7 @@ de_instrument_line(PyCodeObject *code, int i) } } if (instr->op.code == INSTRUMENTED_INSTRUCTION) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = original_opcode; + code->_co_monitoring->per_instruction_opcodes[i] = original_opcode; } else { instr->op.code = original_opcode; @@ -722,7 +724,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + int original_opcode = code->_co_monitoring->per_instruction_opcodes[offset]; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ @@ -738,7 +740,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for snaity check */ - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = 0; + code->_co_monitoring->per_instruction_opcodes[offset] = 0; } @@ -749,7 +751,7 @@ instrument(PyCodeObject *code, int offset) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; opcode = lines->original_opcode; assert(!is_instrumented(opcode)); lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; @@ -775,7 +777,7 @@ instrument_line(PyCodeObject *code, int i) if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; CHECK(lines->original_opcode == 255); if (is_instrumented(opcode)) { lines->original_opcode = opcode; @@ -789,7 +791,7 @@ instrument_line(PyCodeObject *code, int i) } assert(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; - CHECK(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); + CHECK(code->_co_monitoring->lines[i].original_opcode != 0); } static void @@ -802,15 +804,15 @@ instrument_per_instruction(PyCodeObject *code, int i) } CHECK(opcode != 0); if (is_instrumented(opcode)) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = opcode; + code->_co_monitoring->per_instruction_opcodes[i] = opcode; } else { assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } - assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] > 0); + assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); instr->op.code = INSTRUMENTED_INSTRUCTION; } @@ -821,10 +823,10 @@ instruction_has_event(PyCodeObject *code, int offset) _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); return OPCODE_HAS_EVENT[opcode]; @@ -838,7 +840,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(instruction_has_event(code, offset)); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; if (monitoring->tools[offset] == 0) { @@ -847,7 +849,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument(code, offset, event); @@ -858,8 +860,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) #ifndef NDEBUG bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) { - int global_tools = PyInterpreterState_Get()->instrumented_events.tools[event]; - int local_tools = code->_co_instrumentation.monitoring_data->local_instrumentation.tools[event]; + int global_tools = PyInterpreterState_Get()->monitors.tools[event]; + int local_tools = code->_co_monitoring->local_monitors.tools[event]; return tools == ((global_tools | local_tools) & tools); } #endif @@ -867,10 +869,10 @@ bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->line_tools) + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools) { - uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; + uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset]; *toolsptr &= ~tools; if (*toolsptr == 0 ) { de_instrument_line(code, offset); @@ -878,7 +880,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -892,11 +894,11 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->tools + assert(code->_co_monitoring); + if (code->_co_monitoring && + code->_co_monitoring->tools ) { - code->_co_instrumentation.monitoring_data->tools[offset] |= tools; + code->_co_monitoring->tools[offset] |= tools; } else { /* Single tool */ @@ -911,10 +913,10 @@ static void add_line_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->line_tools + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools ) { - code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; + code->_co_monitoring->line_tools[offset] |= tools; } else { /* Single tool */ @@ -928,10 +930,10 @@ static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools ) { - code->_co_instrumentation.monitoring_data->per_instruction_tools[offset] |= tools; + code->_co_monitoring->per_instruction_tools[offset] |= tools; } else { /* Single tool */ @@ -944,10 +946,10 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools) + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools) { - uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->per_instruction_tools[offset]; + uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; *toolsptr &= ~tools; if (*toolsptr == 0 ) { de_instrument_per_instruction(code, offset); @@ -955,7 +957,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_per_instruction(code, offset); @@ -972,7 +974,7 @@ call_one_instrument( { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); - PyObject *instrument = interp->tools[tool].instrument_callables[event]; + PyObject *instrument = interp->monitoring_callables[tool][event]; if (instrument == NULL) { return 0; } @@ -1011,18 +1013,17 @@ static inline int most_significant_bit(uint8_t bits) { static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; + return interp->monitoring_version == code->_co_instrumentation_version; } #ifndef NDEBUG static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { - _Py_Monitors expected = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); - return matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, - expected); + _Py_Monitors expected = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); + return monitors_equals(code->_co_monitoring->active_monitors, expected); } #endif @@ -1033,7 +1034,7 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring->tools) { tools = monitoring->tools[i]; } @@ -1043,10 +1044,10 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; } - tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + tools = code->_co_monitoring->active_monitors.tools[event]; } CHECK(tools_is_subset_for_event(code, event, tools)); - CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); + CHECK((tools & code->_co_monitoring->active_monitors.tools[event]) == tools); return tools; } @@ -1186,9 +1187,9 @@ _Py_call_instrumentation_exc_vector( PyCodeObject *code = frame->f_code; assert(args[1] == NULL); args[1] = (PyObject *)code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + uint8_t tools = code->_co_monitoring->active_monitors.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1237,12 +1238,12 @@ _Py_call_instrumentation_exc2( int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { - _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - assert(instrumentation->monitoring_data != NULL); - assert(instrumentation->monitoring_data->lines != NULL); + _PyCoMonitoringData *monitoring = code->_co_monitoring; + assert(monitoring != NULL); + assert(monitoring->lines != NULL); assert(index >= code->_co_firsttraceable); assert(index < Py_SIZE(code)); - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; + _PyCoLineInstrumentationData *line_data = &monitoring->lines[index]; int8_t line_delta = line_data->line_delta; int line = compute_line(code, index, line_delta); return line; @@ -1256,8 +1257,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = instr - _PyCode_CODE(code); - _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; + _PyCoMonitoringData *monitoring = code->_co_monitoring; + _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; if (tstate->tracing) { goto done; @@ -1265,10 +1266,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); - uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? - code->_co_instrumentation.monitoring_data->line_tools[i] : - (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE] | - code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_LINE] + uint8_t tools = code->_co_monitoring->line_tools != NULL ? + code->_co_monitoring->line_tools[i] : + (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE] ); PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { @@ -1311,7 +1312,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); - _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; if (tstate->tracing) { @@ -1320,8 +1321,8 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] | - code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION] + (interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1361,22 +1362,22 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) PyInterpreterState *is = _PyInterpreterState_Get(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); - PyObject *callback = is->tools[tool_id].instrument_callables[event_id]; - is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); + PyObject *callback = is->monitoring_callables[tool_id][event_id]; + is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); return callback; } static void initialize_tools(PyCodeObject *code) { - uint8_t* tools = code->_co_instrumentation.monitoring_data->tools; + uint8_t* tools = code->_co_monitoring->tools; assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + opcode = code->_co_monitoring->lines[i].original_opcode; } bool instrumented = is_instrumented(opcode); if (instrumented) { @@ -1396,7 +1397,7 @@ initialize_tools(PyCodeObject *code) } assert(event >= 0); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); - tools[i] = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + tools[i] = code->_co_monitoring->active_monitors.tools[event]; CHECK(tools[i] != 0); } else { @@ -1423,7 +1424,7 @@ initialize_tools(PyCodeObject *code) static void initialize_lines(PyCodeObject *code) { - _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; assert(line_data != NULL); int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; @@ -1477,7 +1478,7 @@ initialize_lines(PyCodeObject *code) static void initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) { - uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; + uint8_t *line_tools = code->_co_monitoring->line_tools; assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1489,19 +1490,19 @@ static int allocate_instrumentation_data(PyCodeObject *code) { - if (code->_co_instrumentation.monitoring_data == NULL) { - code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); - if (code->_co_instrumentation.monitoring_data == NULL) { + if (code->_co_monitoring == NULL) { + code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); + if (code->_co_monitoring == NULL) { PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->local_instrumentation = (_Py_Monitors){ 0 }; - code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_Monitors){ 0 }; - code->_co_instrumentation.monitoring_data->tools = NULL; - code->_co_instrumentation.monitoring_data->lines = NULL; - code->_co_instrumentation.monitoring_data->line_tools = NULL; - code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; - code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; + code->_co_monitoring->local_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->active_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->tools = NULL; + code->_co_monitoring->lines = NULL; + code->_co_monitoring->line_tools = NULL; + code->_co_monitoring->per_instruction_opcodes = NULL; + code->_co_monitoring->per_instruction_tools = NULL; } return 0; } @@ -1513,30 +1514,30 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors all_events = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors all_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); bool multitools = multiple_tools(&all_events); - if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { - code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->tools == NULL) { + if (code->_co_monitoring->tools == NULL && multitools) { + code->_co_monitoring->tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->tools == NULL) { PyErr_NoMemory(); return -1; } initialize_tools(code); } if (all_events.tools[PY_MONITORING_EVENT_LINE]) { - if (code->_co_instrumentation.monitoring_data->lines == NULL) { - code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); - if (code->_co_instrumentation.monitoring_data->lines == NULL) { + if (code->_co_monitoring->lines == NULL) { + code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->lines == NULL) { PyErr_NoMemory(); return -1; } initialize_lines(code); } - if (multitools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { - code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { + if (multitools && code->_co_monitoring->line_tools == NULL) { + code->_co_monitoring->line_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->line_tools == NULL) { PyErr_NoMemory(); return -1; } @@ -1544,26 +1545,26 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } } if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + if (code->_co_monitoring->per_instruction_opcodes == NULL) { + code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->per_instruction_opcodes == NULL) { PyErr_NoMemory(); return -1; } /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = 0; + code->_co_monitoring->per_instruction_opcodes[i] = 0; } } - if (multitools && code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { - code->_co_instrumentation.monitoring_data->per_instruction_tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { + code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->per_instruction_tools == NULL) { PyErr_NoMemory(); return -1; } /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data->per_instruction_tools[i] = 0; + code->_co_monitoring->per_instruction_tools[i] = 0; } } } @@ -1599,25 +1600,25 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_Monitors active_events = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors active_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); _Py_Monitors new_events; _Py_Monitors removed_events; - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; + bool restarted = interp->last_restart_version > code->_co_instrumentation_version; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; + removed_events = code->_co_monitoring->active_monitors; new_events = active_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, active_events); - new_events = matrix_sub(active_events, code->_co_instrumentation.monitoring_data->current_instrumentation); - assert(matrix_empty(matrix_and(new_events, removed_events))); + removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events); + new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors); + assert(monitors_empty(monitors_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->current_instrumentation = active_events; - code->_co_instrumentation.monitoring_version = interp->monitoring_version; - if (matrix_empty(new_events) && matrix_empty(removed_events)) { + code->_co_monitoring->active_monitors = active_events; + code->_co_instrumentation_version = interp->monitoring_version; + if (monitors_empty(new_events) && monitors_empty(removed_events)) { #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif @@ -1666,7 +1667,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { - _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { @@ -1751,11 +1752,11 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); - uint32_t existing_events = get_events(&interp->instrumented_events, tool_id); + uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { return; } - set_events(&interp->instrumented_events, tool_id, events); + set_events(&interp->monitors, tool_id, events); interp->monitoring_version++; instrument_all_executing_code_objects(interp); } @@ -1769,7 +1770,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors *local = &code->_co_instrumentation.monitoring_data->local_instrumentation; + _Py_Monitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_events(local, tool_id); if (existing_events == events) { return 0; @@ -1777,7 +1778,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent set_events(local, tool_id, events); if (is_version_up_to_date(code, interp)) { /* Force instrumentation update */ - code->_co_instrumentation.monitoring_version = UINT64_MAX; + code->_co_instrumentation_version = UINT64_MAX; } _Py_Instrument(code, interp); return 0; @@ -1930,7 +1931,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - _Py_Monitors *m = &_PyInterpreterState_Get()->instrumented_events; + _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1990,8 +1991,7 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, } _PyMonitoringEventSet event_set = 0; for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - _Py_Monitors *m = &((PyCodeObject *)code)-> - _co_instrumentation.monitoring_data->local_instrumentation; + _Py_Monitors *m = &((PyCodeObject *)code)->_co_monitoring->local_monitors; if ((m->tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2104,7 +2104,7 @@ monitoring__all_events_impl(PyObject *module) return NULL; } for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - uint8_t tools = interp->instrumented_events.tools[e]; + uint8_t tools = interp->monitors.tools[e]; if (tools == 0) { continue; } diff --git a/Python/pystate.c b/Python/pystate.c index 67c479272631d9..611922df14e299 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -644,11 +644,11 @@ init_interpreter(PyInterpreterState *interp, PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { - interp->instrumented_events.tools[i] = 0; + interp->monitors.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - interp->tools[t].instrument_callables[i] = NULL; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + interp->monitoring_callables[t][e] = NULL; } } interp->f_opcode_trace_set = false; @@ -778,11 +778,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { - interp->instrumented_events.tools[i] = 0; + interp->monitors.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - Py_CLEAR(interp->tools[t].instrument_callables[i]); + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + Py_CLEAR(interp->monitoring_callables[t][e]); } } From 0982e5ea6143ff64d0af90a4aff26f7f4cca409d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 17:04:13 +0000 Subject: [PATCH 052/116] Remove out of dat comments. --- Python/instrumentation.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c0f4d4920cda82..c6a1ee67cd21f2 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -680,7 +680,6 @@ de_instrument(PyCodeObject *code, int offset, int event) static void de_instrument_line(PyCodeObject *code, int i) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_INSTRUCTION) { @@ -718,7 +717,6 @@ de_instrument_line(PyCodeObject *code, int i) static void de_instrument_per_instruction(PyCodeObject *code, int offset) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode != INSTRUMENTED_INSTRUCTION) { From 0693423091aa055ace9c8efa1dc5640ca1b94100 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Mar 2023 19:35:28 +0000 Subject: [PATCH 053/116] Remove instrumentation for JUMP_IF_[FALS|TRUE]_OR_POP --- Lib/opcode.py | 3 +-- Python/bytecodes.c | 30 ------------------------------ Python/instrumentation.c | 10 ---------- 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 91d2afebf08ab6..49f86890fa599a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -240,8 +240,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) -def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) -def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) + def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index df4686fa033a59..1acf2fa55463b3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3325,36 +3325,6 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP, ( -- )) { - PyObject *cond = TOP(); - int err = PyObject_IsTrue(cond); - ERROR_IF(err < 0, error); - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - int shrink = 1 - err; - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - if (shrink) { - STACK_SHRINK(1); - Py_DECREF(cond); - } - } - - inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP, ( -- )) { - PyObject *cond = TOP(); - int err = PyObject_IsTrue(cond); - ERROR_IF(err < 0, error); - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + (1-err)*oparg; - int shrink = err; - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - if (shrink) { - STACK_SHRINK(1); - Py_DECREF(cond); - } - } - inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c6a1ee67cd21f2..b49b8d9380810a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -47,8 +47,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, @@ -86,8 +84,6 @@ static const bool OPCODE_HAS_EVENT[256] = { [COMPARE_AND_BRANCH] = true, [INSTRUMENTED_JUMP_FORWARD] = true, [INSTRUMENTED_JUMP_BACKWARD] = true, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = true, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = true, [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, [INSTRUMENTED_POP_JUMP_IF_NONE] = true, @@ -109,8 +105,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, @@ -138,10 +132,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, From 0ee2aee8fd375b03feea7b5a1035d68b8d97e89b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 00:41:37 +0000 Subject: [PATCH 054/116] Minor fixups from merge. --- Python/bytecodes.c | 11 ++--- Python/generated_cases.c.h | 93 +++++++++++++++++++------------------- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 51b041c85af6f0..6f7729402eda1f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3002,12 +3002,9 @@ dummy_func( } inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) { - if (oparg & 1) { - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -3047,7 +3044,7 @@ dummy_func( else { result = PyObject_Call(func, callargs, kwargs); } - + DECREF_INPUTS(); assert(PEEK(3 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); CHECK_EVAL_BREAKER(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ffe63fe0ddedf0..7089adfb563926 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4221,12 +4221,9 @@ PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; #line 3005 "Python/bytecodes.c" - if (oparg & 1) { - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -4266,10 +4263,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - + #line 4267 "Python/generated_cases.c.h" + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + #line 3048 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4273 "Python/generated_cases.c.h" + #line 4274 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4284,7 +4285,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3061 "Python/bytecodes.c" + #line 3058 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4313,14 +4314,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4317 "Python/generated_cases.c.h" + #line 4318 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3092 "Python/bytecodes.c" + #line 3089 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4341,7 +4342,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4345 "Python/generated_cases.c.h" + #line 4346 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4349,15 +4350,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3115 "Python/bytecodes.c" + #line 3112 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4355 "Python/generated_cases.c.h" + #line 4356 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3117 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4361 "Python/generated_cases.c.h" + #line 4362 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4368,7 +4369,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3121 "Python/bytecodes.c" + #line 3118 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4403,7 +4404,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4407 "Python/generated_cases.c.h" + #line 4408 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4412,10 +4413,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3158 "Python/bytecodes.c" + #line 3155 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4419 "Python/generated_cases.c.h" + #line 4420 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4427,7 +4428,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3163 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4442,12 +4443,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4446 "Python/generated_cases.c.h" + #line 4447 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3178 "Python/bytecodes.c" + #line 3175 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4451 "Python/generated_cases.c.h" + #line 4452 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4457,16 +4458,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3183 "Python/bytecodes.c" + #line 3180 "Python/bytecodes.c" assert(oparg >= 2); - #line 4463 "Python/generated_cases.c.h" + #line 4464 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3187 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4486,11 +4487,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4490 "Python/generated_cases.c.h" + #line 4491 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3209 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4502,26 +4503,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4506 "Python/generated_cases.c.h" + #line 4507 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3224 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4512 "Python/generated_cases.c.h" + #line 4513 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3229 "Python/bytecodes.c" + #line 3226 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4519 "Python/generated_cases.c.h" + #line 4520 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3234 "Python/bytecodes.c" + #line 3231 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4529,12 +4530,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4534 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3244 "Python/bytecodes.c" + #line 3241 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4542,12 +4543,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4546 "Python/generated_cases.c.h" + #line 4547 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3254 "Python/bytecodes.c" + #line 3251 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4560,12 +4561,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4564 "Python/generated_cases.c.h" + #line 4565 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3269 "Python/bytecodes.c" + #line 3266 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4578,22 +4579,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4582 "Python/generated_cases.c.h" + #line 4583 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3284 "Python/bytecodes.c" + #line 3281 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4593 "Python/generated_cases.c.h" + #line 4594 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3292 "Python/bytecodes.c" + #line 3289 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4599 "Python/generated_cases.c.h" + #line 4600 "Python/generated_cases.c.h" } From acdca93f1ce4cb738a2f5b2f9916fcf3df996afb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 01:47:02 +0000 Subject: [PATCH 055/116] Remove instrumentation for COMPARE_AND_BRANCH. --- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 - Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 2 +- Python/bytecodes.c | 22 -- Python/generated_cases.c.h | 401 +++++++++++++-------------- Python/instrumentation.c | 18 -- Python/opcode_metadata.h | 5 - Python/opcode_targets.h | 2 +- 9 files changed, 193 insertions(+), 264 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 1e8a00a2b54c3c..acd8c8d3b1143c 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -141,7 +141,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, @@ -487,7 +486,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", + [239] = "<239>", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", @@ -581,6 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ + case 239: \ case 247: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 5a54888fe0e037..962b61cdf3bb5d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -120,7 +120,6 @@ extern "C" { #define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 236 #define INSTRUMENTED_RETURN_CONST 237 #define INSTRUMENTED_FOR_ITER 238 -#define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 #define INSTRUMENTED_RETURN_VALUE 242 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 91c22a98b809bf..bbd5eb36148ba0 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,7 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index adf65cd629dd0a..ebfc0ee9276d81 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -230,7 +230,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 236) def_op('INSTRUMENTED_RETURN_CONST', 237) def_op('INSTRUMENTED_FOR_ITER', 238) -def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) + def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) def_op('INSTRUMENTED_RETURN_VALUE', 242) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f7729402eda1f..34951e8d8f512e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1819,28 +1819,6 @@ dummy_func( } } - inst(INSTRUMENTED_COMPARE_AND_BRANCH, ( -- )) { - assert((oparg >> 4) <= Py_GE); - PyObject *right = POP(); - PyObject *left = POP(); - PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); - ERROR_IF(cond == NULL, error); - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = 0; - int err = PyObject_IsTrue(cond); - _Py_CODEUNIT *here = next_instr-1; - Py_DECREF(cond); - ERROR_IF(err < 0, error); - if (jump_on_true == (err != 0)) { - offset = _Py_OPARG(next_instr[1]); - } - INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); - } - inst(COMPARE_AND_BRANCH_FLOAT, (unused/2, left, right -- )) { DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7089adfb563926..99f11245cd23cb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2597,35 +2597,10 @@ DISPATCH(); } - TARGET(INSTRUMENTED_COMPARE_AND_BRANCH) { - #line 1823 "Python/bytecodes.c" - assert((oparg >> 4) <= Py_GE); - PyObject *right = POP(); - PyObject *left = POP(); - PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); - if (cond == NULL) goto error; - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = 0; - int err = PyObject_IsTrue(cond); - _Py_CODEUNIT *here = next_instr-1; - Py_DECREF(cond); - if (err < 0) goto error; - if (jump_on_true == (err != 0)) { - offset = _Py_OPARG(next_instr[1]); - } - INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); - #line 2622 "Python/generated_cases.c.h" - DISPATCH(); - } - TARGET(COMPARE_AND_BRANCH_FLOAT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1845 "Python/bytecodes.c" + #line 1823 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); STAT_INC(COMPARE_AND_BRANCH, hit); @@ -2639,7 +2614,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2643 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2648,7 +2623,7 @@ TARGET(COMPARE_AND_BRANCH_INT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1862 "Python/bytecodes.c" + #line 1840 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_AND_BRANCH); @@ -2666,7 +2641,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2670 "Python/generated_cases.c.h" + #line 2645 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2675,7 +2650,7 @@ TARGET(COMPARE_AND_BRANCH_STR) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1883 "Python/bytecodes.c" + #line 1861 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); STAT_INC(COMPARE_AND_BRANCH, hit); @@ -2690,7 +2665,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2694 "Python/generated_cases.c.h" + #line 2669 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2700,14 +2675,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1900 "Python/bytecodes.c" + #line 1878 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2706 "Python/generated_cases.c.h" + #line 2681 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1902 "Python/bytecodes.c" + #line 1880 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2711 "Python/generated_cases.c.h" + #line 2686 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2717,15 +2692,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1906 "Python/bytecodes.c" + #line 1884 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2723 "Python/generated_cases.c.h" + #line 2698 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1908 "Python/bytecodes.c" + #line 1886 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2729 "Python/generated_cases.c.h" + #line 2704 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2736,12 +2711,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1913 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2742 "Python/generated_cases.c.h" + #line 2717 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1915 "Python/bytecodes.c" + #line 1893 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2749,10 +2724,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2753 "Python/generated_cases.c.h" + #line 2728 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1923 "Python/bytecodes.c" + #line 1901 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2761,7 +2736,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2765 "Python/generated_cases.c.h" + #line 2740 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2771,21 +2746,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1934 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2778 "Python/generated_cases.c.h" + #line 2753 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1937 "Python/bytecodes.c" + #line 1915 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2785 "Python/generated_cases.c.h" + #line 2760 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1942 "Python/bytecodes.c" + #line 1920 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2789 "Python/generated_cases.c.h" + #line 2764 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2794,15 +2769,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1946 "Python/bytecodes.c" + #line 1924 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2801 "Python/generated_cases.c.h" + #line 2776 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1949 "Python/bytecodes.c" + #line 1927 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2806 "Python/generated_cases.c.h" + #line 2781 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2811,29 +2786,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1953 "Python/bytecodes.c" + #line 1931 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2819 "Python/generated_cases.c.h" + #line 2794 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1959 "Python/bytecodes.c" + #line 1937 "Python/bytecodes.c" JUMPBY(oparg); - #line 2828 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1963 "Python/bytecodes.c" + #line 1941 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2837 "Python/generated_cases.c.h" + #line 2812 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2841,7 +2816,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1969 "Python/bytecodes.c" + #line 1947 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2851,9 +2826,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2855 "Python/generated_cases.c.h" + #line 2830 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1979 "Python/bytecodes.c" + #line 1957 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2861,14 +2836,14 @@ if (err < 0) goto pop_1_error; } } - #line 2865 "Python/generated_cases.c.h" + #line 2840 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1989 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2878,9 +2853,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2882 "Python/generated_cases.c.h" + #line 2857 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1999 "Python/bytecodes.c" + #line 1977 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2888,67 +2863,67 @@ if (err < 0) goto pop_1_error; } } - #line 2892 "Python/generated_cases.c.h" + #line 2867 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2009 "Python/bytecodes.c" + #line 1987 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2901 "Python/generated_cases.c.h" + #line 2876 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2011 "Python/bytecodes.c" + #line 1989 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2909 "Python/generated_cases.c.h" + #line 2884 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2019 "Python/bytecodes.c" + #line 1997 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2922 "Python/generated_cases.c.h" + #line 2897 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2025 "Python/bytecodes.c" + #line 2003 "Python/bytecodes.c" } - #line 2926 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2029 "Python/bytecodes.c" + #line 2007 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2939 "Python/generated_cases.c.h" + #line 2914 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2038 "Python/bytecodes.c" + #line 2016 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2952 "Python/generated_cases.c.h" + #line 2927 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2959,16 +2934,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2046 "Python/bytecodes.c" + #line 2024 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2968 "Python/generated_cases.c.h" + #line 2943 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2051 "Python/bytecodes.c" + #line 2029 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2976,7 +2951,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2980 "Python/generated_cases.c.h" + #line 2955 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2985,10 +2960,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2061 "Python/bytecodes.c" + #line 2039 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2992 "Python/generated_cases.c.h" + #line 2967 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2998,10 +2973,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2067 "Python/bytecodes.c" + #line 2045 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3005 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3012,11 +2987,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2073 "Python/bytecodes.c" + #line 2051 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3020 "Python/generated_cases.c.h" + #line 2995 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3025,14 +3000,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2079 "Python/bytecodes.c" + #line 2057 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3032 "Python/generated_cases.c.h" + #line 3007 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2082 "Python/bytecodes.c" + #line 2060 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3036 "Python/generated_cases.c.h" + #line 3011 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3040,7 +3015,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2086 "Python/bytecodes.c" + #line 2064 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3063,11 +3038,11 @@ if (iter == NULL) { goto error; } - #line 3067 "Python/generated_cases.c.h" + #line 3042 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2109 "Python/bytecodes.c" + #line 2087 "Python/bytecodes.c" } - #line 3071 "Python/generated_cases.c.h" + #line 3046 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3078,7 +3053,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2128 "Python/bytecodes.c" + #line 2106 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3109,7 +3084,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3113 "Python/generated_cases.c.h" + #line 3088 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3117,7 +3092,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2161 "Python/bytecodes.c" + #line 2139 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3143,14 +3118,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3147 "Python/generated_cases.c.h" + #line 3122 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2189 "Python/bytecodes.c" + #line 2167 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3170,7 +3145,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3174 "Python/generated_cases.c.h" + #line 3149 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3180,7 +3155,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2211 "Python/bytecodes.c" + #line 2189 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3200,7 +3175,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3204 "Python/generated_cases.c.h" + #line 3179 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3210,7 +3185,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2233 "Python/bytecodes.c" + #line 2211 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3228,7 +3203,7 @@ if (next == NULL) { goto error; } - #line 3232 "Python/generated_cases.c.h" + #line 3207 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3237,7 +3212,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2253 "Python/bytecodes.c" + #line 2231 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3252,14 +3227,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3256 "Python/generated_cases.c.h" + #line 3231 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2270 "Python/bytecodes.c" + #line 2248 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3282,16 +3257,16 @@ Py_DECREF(enter); goto error; } - #line 3286 "Python/generated_cases.c.h" + #line 3261 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2293 "Python/bytecodes.c" + #line 2271 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3295 "Python/generated_cases.c.h" + #line 3270 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3303,7 +3278,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2303 "Python/bytecodes.c" + #line 2281 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3329,16 +3304,16 @@ Py_DECREF(enter); goto error; } - #line 3333 "Python/generated_cases.c.h" + #line 3308 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2329 "Python/bytecodes.c" + #line 2307 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3342 "Python/generated_cases.c.h" + #line 3317 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3350,7 +3325,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2338 "Python/bytecodes.c" + #line 2316 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3371,7 +3346,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3375 "Python/generated_cases.c.h" + #line 3350 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3380,7 +3355,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2361 "Python/bytecodes.c" + #line 2339 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3390,7 +3365,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3394 "Python/generated_cases.c.h" + #line 3369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3404,7 +3379,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2373 "Python/bytecodes.c" + #line 2351 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3421,7 +3396,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3400 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3435,7 +3410,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2392 "Python/bytecodes.c" + #line 2370 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3445,7 +3420,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3449 "Python/generated_cases.c.h" + #line 3424 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3459,7 +3434,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2404 "Python/bytecodes.c" + #line 2382 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3473,7 +3448,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3477 "Python/generated_cases.c.h" + #line 3452 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3482,16 +3457,16 @@ } TARGET(KW_NAMES) { - #line 2420 "Python/bytecodes.c" + #line 2398 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3490 "Python/generated_cases.c.h" + #line 3465 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2426 "Python/bytecodes.c" + #line 2404 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3504,7 +3479,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3508 "Python/generated_cases.c.h" + #line 3483 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3514,7 +3489,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2471 "Python/bytecodes.c" + #line 2449 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3595,7 +3570,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3599 "Python/generated_cases.c.h" + #line 3574 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3607,7 +3582,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2558 "Python/bytecodes.c" + #line 2536 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3617,7 +3592,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3621 "Python/generated_cases.c.h" + #line 3596 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3626,7 +3601,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2570 "Python/bytecodes.c" + #line 2548 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3651,7 +3626,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3655 "Python/generated_cases.c.h" + #line 3630 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3660,7 +3635,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2597 "Python/bytecodes.c" + #line 2575 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3690,7 +3665,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3694 "Python/generated_cases.c.h" + #line 3669 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3698,7 +3673,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2629 "Python/bytecodes.c" + #line 2607 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3708,7 +3683,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3712 "Python/generated_cases.c.h" + #line 3687 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3721,7 +3696,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2641 "Python/bytecodes.c" + #line 2619 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3732,7 +3707,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3736 "Python/generated_cases.c.h" + #line 3711 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3746,7 +3721,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2655 "Python/bytecodes.c" + #line 2633 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3757,7 +3732,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3761 "Python/generated_cases.c.h" + #line 3736 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3771,7 +3746,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2669 "Python/bytecodes.c" + #line 2647 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3793,7 +3768,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3797 "Python/generated_cases.c.h" + #line 3772 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3807,7 +3782,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2694 "Python/bytecodes.c" + #line 2672 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3835,7 +3810,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3839 "Python/generated_cases.c.h" + #line 3814 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3849,7 +3824,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2725 "Python/bytecodes.c" + #line 2703 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3881,7 +3856,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3885 "Python/generated_cases.c.h" + #line 3860 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3895,7 +3870,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2760 "Python/bytecodes.c" + #line 2738 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3927,7 +3902,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3931 "Python/generated_cases.c.h" + #line 3906 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3941,7 +3916,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2795 "Python/bytecodes.c" + #line 2773 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3966,7 +3941,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3970 "Python/generated_cases.c.h" + #line 3945 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3979,7 +3954,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2822 "Python/bytecodes.c" + #line 2800 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4006,7 +3981,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4010 "Python/generated_cases.c.h" + #line 3985 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4018,7 +3993,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2852 "Python/bytecodes.c" + #line 2830 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4036,14 +4011,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4040 "Python/generated_cases.c.h" + #line 4015 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2872 "Python/bytecodes.c" + #line 2850 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4074,7 +4049,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4078 "Python/generated_cases.c.h" + #line 4053 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4087,7 +4062,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2906 "Python/bytecodes.c" + #line 2884 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4116,7 +4091,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4120 "Python/generated_cases.c.h" + #line 4095 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4129,7 +4104,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2938 "Python/bytecodes.c" + #line 2916 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4158,7 +4133,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4162 "Python/generated_cases.c.h" + #line 4137 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4171,7 +4146,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2970 "Python/bytecodes.c" + #line 2948 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4199,7 +4174,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4203 "Python/generated_cases.c.h" + #line 4178 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4209,9 +4184,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3001 "Python/bytecodes.c" + #line 2979 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4215 "Python/generated_cases.c.h" + #line 4190 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4220,7 +4195,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3005 "Python/bytecodes.c" + #line 2983 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4263,14 +4238,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4267 "Python/generated_cases.c.h" + #line 4242 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3048 "Python/bytecodes.c" + #line 3026 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4274 "Python/generated_cases.c.h" + #line 4249 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4285,7 +4260,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3058 "Python/bytecodes.c" + #line 3036 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4314,14 +4289,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4318 "Python/generated_cases.c.h" + #line 4293 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3089 "Python/bytecodes.c" + #line 3067 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4342,7 +4317,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4346 "Python/generated_cases.c.h" + #line 4321 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4350,15 +4325,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3112 "Python/bytecodes.c" + #line 3090 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4356 "Python/generated_cases.c.h" + #line 4331 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3114 "Python/bytecodes.c" + #line 3092 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4362 "Python/generated_cases.c.h" + #line 4337 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4369,7 +4344,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3118 "Python/bytecodes.c" + #line 3096 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4404,7 +4379,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4408 "Python/generated_cases.c.h" + #line 4383 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4413,10 +4388,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3155 "Python/bytecodes.c" + #line 3133 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4420 "Python/generated_cases.c.h" + #line 4395 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4428,7 +4403,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3160 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4443,12 +4418,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4447 "Python/generated_cases.c.h" + #line 4422 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3175 "Python/bytecodes.c" + #line 3153 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4452 "Python/generated_cases.c.h" + #line 4427 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4458,16 +4433,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3180 "Python/bytecodes.c" + #line 3158 "Python/bytecodes.c" assert(oparg >= 2); - #line 4464 "Python/generated_cases.c.h" + #line 4439 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3184 "Python/bytecodes.c" + #line 3162 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4487,11 +4462,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4491 "Python/generated_cases.c.h" + #line 4466 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3206 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4503,26 +4478,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4507 "Python/generated_cases.c.h" + #line 4482 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3221 "Python/bytecodes.c" + #line 3199 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4513 "Python/generated_cases.c.h" + #line 4488 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3226 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4520 "Python/generated_cases.c.h" + #line 4495 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3231 "Python/bytecodes.c" + #line 3209 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4530,12 +4505,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4534 "Python/generated_cases.c.h" + #line 4509 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3241 "Python/bytecodes.c" + #line 3219 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4543,12 +4518,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4547 "Python/generated_cases.c.h" + #line 4522 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3251 "Python/bytecodes.c" + #line 3229 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4561,12 +4536,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4565 "Python/generated_cases.c.h" + #line 4540 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3266 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4579,22 +4554,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4583 "Python/generated_cases.c.h" + #line 4558 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3281 "Python/bytecodes.c" + #line 3259 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4594 "Python/generated_cases.c.h" + #line 4569 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3289 "Python/bytecodes.c" + #line 3267 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4600 "Python/generated_cases.c.h" + #line 4575 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index fee0356e19b455..f6e8353fdd14b5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -42,14 +42,12 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, @@ -77,14 +75,12 @@ static const bool OPCODE_HAS_EVENT[256] = { [POP_JUMP_IF_TRUE] = true, [POP_JUMP_IF_NONE] = true, [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, [INSTRUMENTED_JUMP_FORWARD] = true, [INSTRUMENTED_JUMP_BACKWARD] = true, [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, [INSTRUMENTED_POP_JUMP_IF_NONE] = true, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_COMPARE_AND_BRANCH] = true, [INSTRUMENTED_FOR_ITER] = true, [END_FOR] = true, [INSTRUMENTED_END_FOR] = true, @@ -105,7 +101,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, @@ -136,8 +131,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, - [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [END_SEND] = INSTRUMENTED_END_SEND, @@ -357,10 +350,6 @@ Instruction read_instruction(PyCodeObject *code, int offset) int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; /* TO DO -- Use instruction length, not cache count table */ result.length += _PyOpcode_Caches[base]; - if (base == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - result.length += 1; - } return result; } @@ -1638,13 +1627,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i++; base_opcode = _Py_GetBaseOpcode(code, i); } - /* TO DO -- Use instruction length, not cache count */ - if (base_opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - CHECK(instr[2].op.code == POP_JUMP_IF_FALSE || - instr[2].op.code == POP_JUMP_IF_TRUE); - i++; - } i += _PyOpcode_Caches[base_opcode]; } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 7d6eb5bcd29977..df842e5a3316d0 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -231,8 +231,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case COMPARE_AND_BRANCH: return 2; - case INSTRUMENTED_COMPARE_AND_BRANCH: - return 0; case COMPARE_AND_BRANCH_FLOAT: return 2; case COMPARE_AND_BRANCH_INT: @@ -617,8 +615,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case COMPARE_AND_BRANCH: return 0; - case INSTRUMENTED_COMPARE_AND_BRANCH: - return 0; case COMPARE_AND_BRANCH_FLOAT: return 0; case COMPARE_AND_BRANCH_INT: @@ -896,7 +892,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000 }, [COMPARE_OP] = { true, INSTR_FMT_IBC }, [COMPARE_AND_BRANCH] = { true, INSTR_FMT_IBC0 }, - [INSTRUMENTED_COMPARE_AND_BRANCH] = { true, INSTR_FMT_IB }, [COMPARE_AND_BRANCH_FLOAT] = { true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_INT] = { true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_STR] = { true, INSTR_FMT_IBC0 }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9018e84e805793..146e41c0a00432 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -238,7 +238,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_RETURN_VALUE, From e7f6c374deb32642aa3fd423e8ea149534cda9b0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 02:02:23 +0000 Subject: [PATCH 056/116] Group instrumented opcodes. --- Include/internal/pycore_opcode.h | 16 ++++++++-------- Include/opcode.h | 8 ++++---- Lib/opcode.py | 11 ++++------- Python/opcode_targets.h | 10 +++++----- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 344b3da287f68f..0ebad1ae45623c 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -480,11 +480,11 @@ static const char *const _PyOpcode_OpName[263] = { [232] = "<232>", [233] = "<233>", [234] = "<234>", + [235] = "<235>", + [236] = "<236>", + [237] = "<237>", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [239] = "<239>", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", @@ -492,8 +492,8 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [247] = "<247>", - [248] = "<248>", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", @@ -579,9 +579,9 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 239: \ - case 247: \ - case 248: \ + case 235: \ + case 236: \ + case 237: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index ea5646d25e6d3a..fcca0daa6de179 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -115,10 +115,8 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 -#define INSTRUMENTED_POP_JUMP_IF_NONE 235 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 236 -#define INSTRUMENTED_RETURN_CONST 237 -#define INSTRUMENTED_FOR_ITER 238 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 #define INSTRUMENTED_RETURN_VALUE 242 @@ -126,6 +124,8 @@ extern "C" { #define INSTRUMENTED_CALL_FUNCTION_EX 244 #define INSTRUMENTED_JUMP_FORWARD 245 #define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_END_FOR 251 diff --git a/Lib/opcode.py b/Lib/opcode.py index 5f13ca6f0a8ed8..2057d451f0eb26 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -224,11 +224,8 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions -def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 235) -def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 236) -def_op('INSTRUMENTED_RETURN_CONST', 237) -def_op('INSTRUMENTED_FOR_ITER', 238) - +def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) +def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) def_op('INSTRUMENTED_RETURN_VALUE', 242) @@ -236,12 +233,12 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) - +def_op('INSTRUMENTED_RETURN_CONST', 247) +def_op('INSTRUMENTED_FOR_ITER', 248) def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) def_op('INSTRUMENTED_END_SEND', 252) - def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) # 255 is reserved diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index b03da52b61271f..a9ab58a42beaf9 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -234,11 +234,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_FOR_ITER, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_RETURN_VALUE, @@ -246,8 +246,8 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_RETURN_CONST, + &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_END_FOR, From e44ebc59af7dc645acc918d0e48283087c1cbe5f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 03:17:26 +0000 Subject: [PATCH 057/116] Remove last references to DO_TRACING --- Include/internal/pycore_opcode.h | 3 ++- Include/opcode.h | 1 - Python/ceval.c | 33 -------------------------------- Python/makeopcodetargets.py | 1 - Python/opcode_targets.h | 2 +- Tools/build/generate_opcode_h.py | 3 --- 6 files changed, 3 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 0ebad1ae45623c..53a446e378d143 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -500,7 +500,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [DO_TRACING] = "DO_TRACING", + [255] = "<255>", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_WITH] = "SETUP_WITH", @@ -582,6 +582,7 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ + case 255: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index fcca0daa6de179..e619ed968a8e0a 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -203,7 +203,6 @@ extern "C" { #define UNPACK_SEQUENCE_TUPLE 154 #define UNPACK_SEQUENCE_TWO_TUPLE 158 #define SEND_GEN 159 -#define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Python/ceval.c b/Python/ceval.c index 7b9a70e07225a3..004410fe4dc1e6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -802,39 +802,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" -#if USE_COMPUTED_GOTOS - TARGET_DO_TRACING: -#else - case DO_TRACING: -#endif - { - assert(0); - NEXTOPARG(); - PRE_DISPATCH_GOTO(); - // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: - while (opcode == EXTENDED_ARG) { - // CPython hasn't ever traced the instruction after an EXTENDED_ARG. - // Inline the EXTENDED_ARG here, so we can avoid branching there: - INSTRUCTION_START(EXTENDED_ARG); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - // Make sure the next instruction isn't a RESUME, since that needs - // to trace properly (and shouldn't have an EXTENDED_ARG, anyways): - assert(opcode != RESUME); - PRE_DISPATCH_GOTO(); - } - opcode = _PyOpcode_Deopt[opcode]; - if (_PyOpcode_Caches[opcode]) { - uint16_t *counter = &next_instr[1].cache; - // The instruction is going to decrement the counter, so we need to - // increment it here to make sure it doesn't try to specialize: - if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { - INCREMENT_ADAPTIVE_COUNTER(*counter); - } - } - DISPATCH_GOTO(); - } - #if USE_COMPUTED_GOTOS _unknown_opcode: #else diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 33a4b4a76a1253..5aa31803397ce4 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -32,7 +32,6 @@ def write_contents(f): """ opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 - targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): if not opcode.is_pseudo(op): targets[op] = "TARGET_%s" % opname diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index a9ab58a42beaf9..81882e08ebb9d1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -254,5 +254,5 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, - &&TARGET_DO_TRACING + &&_unknown_opcode }; diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 9b2112f7f5f31d..7aaef6b16f3443 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -105,9 +105,6 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna specialized_opmap[name] = next_op opname_including_specialized[next_op] = name used[next_op] = True - specialized_opmap['DO_TRACING'] = 255 - opname_including_specialized[255] = 'DO_TRACING' - used[255] = True with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: fobj.write(header) From e88921e2ccc3156a643257a30873bd6e2a59ab93 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:32:22 +0000 Subject: [PATCH 058/116] Remove unused function. --- Python/instrumentation.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f6e8353fdd14b5..243490d517ec8b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -289,26 +289,6 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } -uint8_t deinstrument(PyCodeObject *code, int offset) -{ - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - return opcode; -} - typedef struct _Instruction { uint8_t extended_args; uint8_t deinstrumented_opcode; From 2d9f22cbe3855ee2f77737b27d761de47e62d450 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:38:37 +0000 Subject: [PATCH 059/116] Streamline de-instrumentation a little. --- Python/instrumentation.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 243490d517ec8b..d5ed9d3c1bce28 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -602,7 +602,7 @@ de_instrument(PyCodeObject *code, int offset, int event) if (opcode != INSTRUMENTED_LINE) { int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { - code->_co_monitoring->per_instruction_opcodes[offset] = deinstrumented; + code->_co_monitoring->per_instruction_opcodes[offset] = opcode = deinstrumented; } break; } @@ -613,7 +613,7 @@ de_instrument(PyCodeObject *code, int offset, int event) int orignal_opcode = lines->original_opcode; int deinstrumented = DE_INSTRUMENT[orignal_opcode]; if (deinstrumented) { - lines->original_opcode = deinstrumented; + lines->original_opcode = opcode = deinstrumented; } break; } @@ -621,12 +621,12 @@ de_instrument(PyCodeObject *code, int offset, int event) { int deinstrumented = DE_INSTRUMENT[opcode]; assert(deinstrumented); - instr->op.code = deinstrumented; + instr->op.code = opcode = deinstrumented; } } - int base_opcode = _Py_GetBaseOpcode(code, offset); + assert(!is_instrumented(opcode)); + int base_opcode = _PyOpcode_Deopt[opcode]; assert(base_opcode != 0); - assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { instr[1].cache = adaptive_counter_warmup(); } From b7579ac8be7ee92e9c7a8096f2e8ab31dda75ea1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:44:53 +0000 Subject: [PATCH 060/116] Use modern API for saving and restoring exception. --- Python/instrumentation.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index d5ed9d3c1bce28..06e65b48eb0d29 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1135,8 +1135,7 @@ _Py_call_instrumentation_exc_vector( { assert(_PyErr_Occurred(tstate)); assert(args[0] == NULL); - PyObject *type, *value, *traceback; - _PyErr_Fetch(tstate, &type, &value, &traceback); + PyObject *exc = _PyErr_GetRaisedException(tstate); PyCodeObject *code = frame->f_code; assert(args[1] == NULL); args[1] = (PyObject *)code; @@ -1155,12 +1154,10 @@ _Py_call_instrumentation_exc_vector( Py_DECREF(offset_obj); } if (err) { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); + Py_XDECREF(exc); } else { - _PyErr_Restore(tstate, type, value, traceback); + _PyErr_SetRaisedException(tstate, exc); } assert(_PyErr_Occurred(tstate)); } From 5a089a64b725170c04839fa188a9016922e99406 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 02:17:21 +0000 Subject: [PATCH 061/116] Refactor instrumentation calls to reduce duplication. --- Python/instrumentation.c | 113 +++++++++++++-------------------------- 1 file changed, 37 insertions(+), 76 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 06e65b48eb0d29..76cb031f27c198 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -988,15 +988,15 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; - if (monitoring->tools) { + if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + assert(event == PY_MONITORING_EVENT_C_RAISE || + event == PY_MONITORING_EVENT_C_RETURN); + event = PY_MONITORING_EVENT_CALL; + } + if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring->tools) { tools = monitoring->tools[i]; } else { - if (event >= PY_MONITORING_UNGROUPED_EVENTS) { - assert(event == PY_MONITORING_EVENT_C_RAISE || - event == PY_MONITORING_EVENT_C_RETURN); - event = PY_MONITORING_EVENT_CALL; - } tools = code->_co_monitoring->active_monitors.tools[event]; } CHECK(tools_is_subset_for_event(code, event, tools)); @@ -1034,44 +1034,48 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -int -_Py_call_instrumentation( +static int +call_instrumentation_vector( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { + assert(!_PyErr_Occurred(tstate)); + assert(args[0] == NULL); PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); + assert(args[1] == NULL); + args[1] = (PyObject *)code; int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { return -1; } + assert(args[2] == NULL); + args[2] = offset_obj; uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + int err = call_instrument(tstate, code, event, &args[1], nargsf | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(offset_obj); return err; } +int +_Py_call_instrumentation( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + PyObject *args[3] = { NULL, NULL, NULL }; + return call_instrumentation_vector(tstate, event, frame, instr, 2, args); +} + int _Py_call_instrumentation_arg( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { - PyCodeObject *code = frame->f_code; - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); - if (offset_obj == NULL) { - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); - Py_DECREF(offset_obj); - return err; + PyObject *args[4] = { NULL, NULL, NULL, arg }; + return call_instrumentation_vector(tstate, event, frame, instr, 3, args); } int @@ -1079,19 +1083,8 @@ _Py_call_instrumentation_2args( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { - PyCodeObject *code = frame->f_code; - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); - if (offset_obj == NULL) { - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[5] = { NULL, (PyObject *)code, offset_obj, arg0, arg1 }; - int err = call_instrument(tstate, code, event, &args[1], 4 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); - Py_DECREF(offset_obj); - return err; + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + return call_instrumentation_vector(tstate, event, frame, instr, 4, args); } int @@ -1104,55 +1097,25 @@ _Py_call_instrumentation_jump( assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int offset = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); PyObject *to_obj = PyLong_FromLong(to); if (to_obj == NULL) { return -1; } - PyObject *from_obj = PyLong_FromLong(offset); - if (from_obj == NULL) { - Py_DECREF(to_obj); - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + PyObject *args[4] = { NULL, NULL, NULL, to_obj }; + int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); Py_DECREF(to_obj); - Py_DECREF(from_obj); - if (err) { - /* Error handling expects next_instr to point to instruction + 1. - * So we add one here. */ - frame->prev_instr++; - } return err; } -void -_Py_call_instrumentation_exc_vector( +static void +call_instrumentation_vector_protected( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); - assert(args[0] == NULL); PyObject *exc = _PyErr_GetRaisedException(tstate); - PyCodeObject *code = frame->f_code; - assert(args[1] == NULL); - args[1] = (PyObject *)code; - assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); - int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_monitoring->active_monitors.tools[event]; - PyObject *offset_obj = PyLong_FromSsize_t(offset); - int err; - if (offset_obj == NULL) { - err = -1; - } - else { - assert(args[2] == NULL); - args[2] = offset_obj; - err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); - Py_DECREF(offset_obj); - } + int err = call_instrumentation_vector(tstate, event, frame, instr, nargsf, args); if (err) { Py_XDECREF(exc); } @@ -1169,8 +1132,7 @@ _Py_call_instrumentation_exc0( { assert(_PyErr_Occurred(tstate)); PyObject *args[3] = { NULL, NULL, NULL }; - Py_ssize_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); + call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); } void @@ -1180,8 +1142,7 @@ _Py_call_instrumentation_exc2( { assert(_PyErr_Occurred(tstate)); PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; - Py_ssize_t nargsf = 4 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); + call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args); } From d5fdec8339b9064ce20817c250f38f6a531288ef Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 03:34:56 +0000 Subject: [PATCH 062/116] Fix refleaks --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0f500cb802daa0..f392266edfdbdd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3183,6 +3183,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); @@ -3193,6 +3194,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 107aac46019491..b7064a71ce3d09 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4473,30 +4473,32 @@ #line 3184 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4482 "Python/generated_cases.c.h" + #line 4483 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3194 "Python/bytecodes.c" + #line 3195 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4495 "Python/generated_cases.c.h" + #line 4497 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3204 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4509,12 +4511,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4513 "Python/generated_cases.c.h" + #line 4515 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3219 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4527,22 +4529,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4531 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3234 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4542 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3242 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4548 "Python/generated_cases.c.h" + #line 4550 "Python/generated_cases.c.h" } From c88741df3751fd6b91b9d8601eaa37d04976d9f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:13:44 +0000 Subject: [PATCH 063/116] Remove commented out code. --- Modules/_lsprof.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 47a917bff843dc..83d034ae7eed78 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -126,10 +126,6 @@ call_timer(ProfilerObject *pObj) static PyObject * normalizeUserObj(PyObject *obj) { - //if (Py_TYPE(obj) == &PyMethodDescr_Type) { - // assert(0); - // return PyObject_Repr(obj); - //} PyCFunctionObject *fn; if (!PyCFunction_Check(obj)) { return Py_NewRef(obj); From b6744cac1d5831144cd97b03ca4b4d21854e6cdb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:15:02 +0000 Subject: [PATCH 064/116] Make sure that stacktop == -1 when stack_pointer is cached. --- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 966 +++++++++++++++++++------------------ 2 files changed, 487 insertions(+), 483 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f392266edfdbdd..fd213f8e7282e7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -99,11 +99,12 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { @@ -3139,6 +3140,7 @@ dummy_func( int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b7064a71ce3d09..6531f4fd764ca6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,23 +21,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } - #line 36 "Python/generated_cases.c.h" + #line 37 "Python/generated_cases.c.h" DISPATCH(); } TARGET(RESUME) { - #line 116 "Python/bytecodes.c" + #line 117 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -49,18 +50,18 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } - #line 53 "Python/generated_cases.c.h" + #line 54 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 130 "Python/bytecodes.c" + #line 131 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 64 "Python/generated_cases.c.h" + #line 65 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -68,11 +69,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 137 "Python/bytecodes.c" + #line 138 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 76 "Python/generated_cases.c.h" + #line 77 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -80,11 +81,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 88 "Python/generated_cases.c.h" + #line 89 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -93,10 +94,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 100 "Python/generated_cases.c.h" + #line 101 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -104,9 +105,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 110 "Python/generated_cases.c.h" + #line 111 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -116,21 +117,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 124 "Python/generated_cases.c.h" + #line 125 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 134 "Python/generated_cases.c.h" + #line 135 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -144,20 +145,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 152 "Python/generated_cases.c.h" + #line 153 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 161 "Python/generated_cases.c.h" + #line 162 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -170,18 +171,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 176 "Python/generated_cases.c.h" + #line 177 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 185 "Python/generated_cases.c.h" + #line 186 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -193,16 +194,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 199 "Python/generated_cases.c.h" + #line 200 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 206 "Python/generated_cases.c.h" + #line 207 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -213,20 +214,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 220 "Python/generated_cases.c.h" + #line 221 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 230 "Python/generated_cases.c.h" + #line 231 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -237,8 +238,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 164 "Python/bytecodes.c" - #line 242 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 243 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -246,9 +247,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 168 "Python/bytecodes.c" + #line 169 "Python/bytecodes.c" res = NULL; - #line 252 "Python/generated_cases.c.h" + #line 253 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -259,14 +260,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 164 "Python/bytecodes.c" - #line 264 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 265 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 164 "Python/bytecodes.c" - #line 270 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 271 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -276,7 +277,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 174 "Python/bytecodes.c" + #line 175 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -286,7 +287,7 @@ } PyErr_SetRaisedException(NULL); } - #line 290 "Python/generated_cases.c.h" + #line 291 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -296,9 +297,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 187 "Python/bytecodes.c" + #line 188 "Python/bytecodes.c" Py_DECREF(receiver); - #line 302 "Python/generated_cases.c.h" + #line 303 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -307,7 +308,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 191 "Python/bytecodes.c" + #line 192 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -316,7 +317,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 320 "Python/generated_cases.c.h" + #line 321 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -325,13 +326,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 202 "Python/bytecodes.c" + #line 203 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 331 "Python/generated_cases.c.h" + #line 332 "Python/generated_cases.c.h" Py_DECREF(value); - #line 204 "Python/bytecodes.c" + #line 205 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 335 "Python/generated_cases.c.h" + #line 336 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -339,11 +340,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 208 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 345 "Python/generated_cases.c.h" + #line 346 "Python/generated_cases.c.h" Py_DECREF(value); - #line 210 "Python/bytecodes.c" + #line 211 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -352,7 +353,7 @@ res = Py_False; } Py_INCREF(res); - #line 356 "Python/generated_cases.c.h" + #line 357 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -360,13 +361,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 221 "Python/bytecodes.c" + #line 222 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 366 "Python/generated_cases.c.h" + #line 367 "Python/generated_cases.c.h" Py_DECREF(value); - #line 223 "Python/bytecodes.c" + #line 224 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 370 "Python/generated_cases.c.h" + #line 371 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -375,7 +376,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 240 "Python/bytecodes.c" + #line 241 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -383,7 +384,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 387 "Python/generated_cases.c.h" + #line 388 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -394,14 +395,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 250 "Python/bytecodes.c" + #line 251 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 405 "Python/generated_cases.c.h" + #line 406 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -412,7 +413,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 259 "Python/bytecodes.c" + #line 260 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -420,7 +421,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 424 "Python/generated_cases.c.h" + #line 425 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -431,13 +432,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 269 "Python/bytecodes.c" + #line 270 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 441 "Python/generated_cases.c.h" + #line 442 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -448,7 +449,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 277 "Python/bytecodes.c" + #line 278 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -456,7 +457,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 460 "Python/generated_cases.c.h" + #line 461 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -466,7 +467,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 293 "Python/bytecodes.c" + #line 294 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -493,7 +494,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 497 "Python/generated_cases.c.h" + #line 498 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -502,14 +503,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 322 "Python/bytecodes.c" + #line 323 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 513 "Python/generated_cases.c.h" + #line 514 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -520,7 +521,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 331 "Python/bytecodes.c" + #line 332 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -528,7 +529,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 532 "Python/generated_cases.c.h" + #line 533 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -541,7 +542,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 349 "Python/bytecodes.c" + #line 350 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -553,12 +554,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 557 "Python/generated_cases.c.h" + #line 558 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 361 "Python/bytecodes.c" + #line 362 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 562 "Python/generated_cases.c.h" + #line 563 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -570,7 +571,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 365 "Python/bytecodes.c" + #line 366 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -583,7 +584,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 587 "Python/generated_cases.c.h" + #line 588 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -594,7 +595,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 380 "Python/bytecodes.c" + #line 381 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -607,7 +608,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 611 "Python/generated_cases.c.h" + #line 612 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -616,7 +617,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 395 "Python/bytecodes.c" + #line 396 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -630,7 +631,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 634 "Python/generated_cases.c.h" + #line 635 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -641,7 +642,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 411 "Python/bytecodes.c" + #line 412 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -655,7 +656,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 659 "Python/generated_cases.c.h" + #line 660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -666,7 +667,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 427 "Python/bytecodes.c" + #line 428 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -674,14 +675,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 678 "Python/generated_cases.c.h" + #line 679 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 435 "Python/bytecodes.c" + #line 436 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 685 "Python/generated_cases.c.h" + #line 686 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -695,7 +696,7 @@ PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); - #line 442 "Python/bytecodes.c" + #line 443 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -714,15 +715,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 718 "Python/generated_cases.c.h" + #line 719 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 463 "Python/bytecodes.c" + #line 464 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 726 "Python/generated_cases.c.h" + #line 727 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -731,13 +732,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 468 "Python/bytecodes.c" + #line 469 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 737 "Python/generated_cases.c.h" + #line 738 "Python/generated_cases.c.h" Py_DECREF(v); - #line 470 "Python/bytecodes.c" + #line 471 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" + #line 742 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -750,7 +751,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 481 "Python/bytecodes.c" + #line 482 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -765,13 +766,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 769 "Python/generated_cases.c.h" + #line 770 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 496 "Python/bytecodes.c" + #line 497 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 775 "Python/generated_cases.c.h" + #line 776 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -781,7 +782,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 500 "Python/bytecodes.c" + #line 501 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -798,7 +799,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 802 "Python/generated_cases.c.h" + #line 803 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -808,13 +809,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 519 "Python/bytecodes.c" + #line 520 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 818 "Python/generated_cases.c.h" + #line 819 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -823,15 +824,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 527 "Python/bytecodes.c" + #line 528 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 830 "Python/generated_cases.c.h" + #line 831 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 530 "Python/bytecodes.c" + #line 531 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 835 "Python/generated_cases.c.h" + #line 836 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -839,14 +840,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 534 "Python/bytecodes.c" + #line 535 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 846 "Python/generated_cases.c.h" + #line 847 "Python/generated_cases.c.h" Py_DECREF(value); - #line 537 "Python/bytecodes.c" + #line 538 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 850 "Python/generated_cases.c.h" + #line 851 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -855,15 +856,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 541 "Python/bytecodes.c" + #line 542 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 862 "Python/generated_cases.c.h" + #line 863 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 544 "Python/bytecodes.c" + #line 545 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 867 "Python/generated_cases.c.h" + #line 868 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -871,7 +872,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 548 "Python/bytecodes.c" + #line 549 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -889,12 +890,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 893 "Python/generated_cases.c.h" + #line 894 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 568 "Python/bytecodes.c" + #line 569 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -905,12 +906,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 909 "Python/generated_cases.c.h" + #line 910 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 581 "Python/bytecodes.c" + #line 582 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -922,12 +923,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 926 "Python/generated_cases.c.h" + #line 927 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 595 "Python/bytecodes.c" + #line 596 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -943,11 +944,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 947 "Python/generated_cases.c.h" + #line 948 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 613 "Python/bytecodes.c" + #line 614 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -960,11 +961,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 964 "Python/generated_cases.c.h" + #line 965 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 628 "Python/bytecodes.c" + #line 629 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); int err = _Py_call_instrumentation_arg( @@ -981,13 +982,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 985 "Python/generated_cases.c.h" + #line 986 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 647 "Python/bytecodes.c" + #line 648 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1000,16 +1001,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1004 "Python/generated_cases.c.h" + #line 1005 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 660 "Python/bytecodes.c" + #line 661 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1011 "Python/generated_cases.c.h" + #line 1012 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 665 "Python/bytecodes.c" + #line 666 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1022,7 +1023,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1026 "Python/generated_cases.c.h" + #line 1027 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1030,7 +1031,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 680 "Python/bytecodes.c" + #line 681 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1074,7 +1075,7 @@ } } - #line 1078 "Python/generated_cases.c.h" + #line 1079 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1085,16 +1086,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 727 "Python/bytecodes.c" + #line 728 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1096 "Python/generated_cases.c.h" + #line 1097 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 734 "Python/bytecodes.c" + #line 735 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1112,7 +1113,7 @@ if (iter == NULL) goto pop_1_error; - #line 1116 "Python/generated_cases.c.h" + #line 1117 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1123,7 +1124,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 760 "Python/bytecodes.c" + #line 761 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1169,7 +1170,7 @@ } } Py_DECREF(v); - #line 1173 "Python/generated_cases.c.h" + #line 1174 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1178,7 +1179,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 808 "Python/bytecodes.c" + #line 809 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1193,12 +1194,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1197 "Python/generated_cases.c.h" + #line 1198 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 825 "Python/bytecodes.c" + #line 826 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1216,12 +1217,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1220 "Python/generated_cases.c.h" + #line 1221 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 845 "Python/bytecodes.c" + #line 846 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1238,15 +1239,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1242 "Python/generated_cases.c.h" + #line 1243 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 864 "Python/bytecodes.c" + #line 865 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1250 "Python/generated_cases.c.h" + #line 1251 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1254,7 +1255,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 869 "Python/bytecodes.c" + #line 870 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1272,26 +1273,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1276 "Python/generated_cases.c.h" + #line 1277 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 889 "Python/bytecodes.c" + #line 890 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1285 "Python/generated_cases.c.h" + #line 1286 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 892 "Python/bytecodes.c" + #line 893 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1295 "Python/generated_cases.c.h" + #line 1296 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1302,23 +1303,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 901 "Python/bytecodes.c" + #line 902 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1311 "Python/generated_cases.c.h" + #line 1312 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 906 "Python/bytecodes.c" + #line 907 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1322 "Python/generated_cases.c.h" + #line 1323 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1327,9 +1328,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 915 "Python/bytecodes.c" + #line 916 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1333 "Python/generated_cases.c.h" + #line 1334 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1337,7 +1338,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 919 "Python/bytecodes.c" + #line 920 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1359,7 +1360,7 @@ if (true) goto error; } } - #line 1363 "Python/generated_cases.c.h" + #line 1364 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1367,33 +1368,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 943 "Python/bytecodes.c" + #line 944 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1378 "Python/generated_cases.c.h" + #line 1379 "Python/generated_cases.c.h" Py_DECREF(v); - #line 950 "Python/bytecodes.c" + #line 951 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1387 "Python/generated_cases.c.h" + #line 1388 "Python/generated_cases.c.h" Py_DECREF(v); - #line 957 "Python/bytecodes.c" + #line 958 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1391 "Python/generated_cases.c.h" + #line 1392 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 961 "Python/bytecodes.c" + #line 962 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1410,7 +1411,7 @@ name); goto error; } - #line 1414 "Python/generated_cases.c.h" + #line 1415 "Python/generated_cases.c.h" DISPATCH(); } @@ -1418,7 +1419,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 987 "Python/bytecodes.c" + #line 988 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1431,11 +1432,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1435 "Python/generated_cases.c.h" + #line 1436 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1000 "Python/bytecodes.c" + #line 1001 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1439 "Python/generated_cases.c.h" + #line 1440 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1445,14 +1446,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1004 "Python/bytecodes.c" + #line 1005 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1456 "Python/generated_cases.c.h" + #line 1457 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1463,7 +1464,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1014 "Python/bytecodes.c" + #line 1015 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1471,7 +1472,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1475 "Python/generated_cases.c.h" + #line 1476 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1482,7 +1483,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1025 "Python/bytecodes.c" + #line 1026 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1490,7 +1491,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1494 "Python/generated_cases.c.h" + #line 1495 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1500,15 +1501,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1036 "Python/bytecodes.c" + #line 1037 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1508 "Python/generated_cases.c.h" + #line 1509 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1040 "Python/bytecodes.c" + #line 1041 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1512 "Python/generated_cases.c.h" + #line 1513 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1519,7 +1520,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1051 "Python/bytecodes.c" + #line 1052 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1535,12 +1536,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1539 "Python/generated_cases.c.h" + #line 1540 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1067 "Python/bytecodes.c" + #line 1068 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1544 "Python/generated_cases.c.h" + #line 1545 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1548,34 +1549,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1071 "Python/bytecodes.c" + #line 1072 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1555 "Python/generated_cases.c.h" + #line 1556 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1074 "Python/bytecodes.c" + #line 1075 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1559 "Python/generated_cases.c.h" + #line 1560 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1078 "Python/bytecodes.c" + #line 1079 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1569 "Python/generated_cases.c.h" + #line 1570 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1081 "Python/bytecodes.c" + #line 1082 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1573 "Python/generated_cases.c.h" + #line 1574 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1085 "Python/bytecodes.c" + #line 1086 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1587,13 +1588,13 @@ } goto error; } - #line 1591 "Python/generated_cases.c.h" + #line 1592 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1099 "Python/bytecodes.c" + #line 1100 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1652,7 +1653,7 @@ } } } - #line 1656 "Python/generated_cases.c.h" + #line 1657 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1663,7 +1664,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1166 "Python/bytecodes.c" + #line 1167 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1715,7 +1716,7 @@ } } null = NULL; - #line 1719 "Python/generated_cases.c.h" + #line 1720 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1729,7 +1730,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1220 "Python/bytecodes.c" + #line 1221 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1740,7 +1741,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1744 "Python/generated_cases.c.h" + #line 1745 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1755,7 +1756,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1233 "Python/bytecodes.c" + #line 1234 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1769,7 +1770,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1773 "Python/generated_cases.c.h" + #line 1774 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1779,16 +1780,16 @@ } TARGET(DELETE_FAST) { - #line 1249 "Python/bytecodes.c" + #line 1250 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1787 "Python/generated_cases.c.h" + #line 1788 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1255 "Python/bytecodes.c" + #line 1256 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1797,12 +1798,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1801 "Python/generated_cases.c.h" + #line 1802 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1266 "Python/bytecodes.c" + #line 1267 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1813,13 +1814,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1817 "Python/generated_cases.c.h" + #line 1818 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1279 "Python/bytecodes.c" + #line 1280 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1851,7 +1852,7 @@ } Py_INCREF(value); } - #line 1855 "Python/generated_cases.c.h" + #line 1856 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1859,7 +1860,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1313 "Python/bytecodes.c" + #line 1314 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1867,7 +1868,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1871 "Python/generated_cases.c.h" + #line 1872 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1875,18 +1876,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1323 "Python/bytecodes.c" + #line 1324 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1884 "Python/generated_cases.c.h" + #line 1885 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1330 "Python/bytecodes.c" + #line 1331 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1897,22 +1898,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1901 "Python/generated_cases.c.h" + #line 1902 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1343 "Python/bytecodes.c" + #line 1344 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1910 "Python/generated_cases.c.h" + #line 1911 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1345 "Python/bytecodes.c" + #line 1346 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1916 "Python/generated_cases.c.h" + #line 1917 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1922,10 +1923,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1349 "Python/bytecodes.c" + #line 1350 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1929 "Python/generated_cases.c.h" + #line 1930 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1935,10 +1936,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1354 "Python/bytecodes.c" + #line 1355 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1942 "Python/generated_cases.c.h" + #line 1943 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1948,7 +1949,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1359 "Python/bytecodes.c" + #line 1360 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1959,13 +1960,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1963 "Python/generated_cases.c.h" + #line 1964 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1370 "Python/bytecodes.c" + #line 1371 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1969 "Python/generated_cases.c.h" + #line 1970 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1974,13 +1975,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1377 "Python/bytecodes.c" + #line 1378 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1980 "Python/generated_cases.c.h" + #line 1981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1379 "Python/bytecodes.c" + #line 1380 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1984 "Python/generated_cases.c.h" + #line 1985 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1988,7 +1989,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1383 "Python/bytecodes.c" + #line 1384 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2003,7 +2004,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2007 "Python/generated_cases.c.h" + #line 2008 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2013,7 +2014,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1400 "Python/bytecodes.c" + #line 1401 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2021,13 +2022,13 @@ if (map == NULL) goto error; - #line 2025 "Python/generated_cases.c.h" + #line 2026 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1408 "Python/bytecodes.c" + #line 1409 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2031 "Python/generated_cases.c.h" + #line 2032 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2035,7 +2036,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1412 "Python/bytecodes.c" + #line 1413 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2075,7 +2076,7 @@ Py_DECREF(ann_dict); } } - #line 2079 "Python/generated_cases.c.h" + #line 2080 "Python/generated_cases.c.h" DISPATCH(); } @@ -2083,7 +2084,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1454 "Python/bytecodes.c" + #line 1455 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2093,14 +2094,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2097 "Python/generated_cases.c.h" + #line 2098 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1464 "Python/bytecodes.c" + #line 1465 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2104 "Python/generated_cases.c.h" + #line 2105 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2108,7 +2109,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1468 "Python/bytecodes.c" + #line 1469 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2116,12 +2117,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2120 "Python/generated_cases.c.h" + #line 2121 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1476 "Python/bytecodes.c" + #line 1477 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2125 "Python/generated_cases.c.h" + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2129,17 +2130,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1482 "Python/bytecodes.c" + #line 1483 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2138 "Python/generated_cases.c.h" + #line 2139 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1487 "Python/bytecodes.c" + #line 1488 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2143 "Python/generated_cases.c.h" + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2149,13 +2150,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1494 "Python/bytecodes.c" + #line 1495 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2159 "Python/generated_cases.c.h" + #line 2160 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2167,7 +2168,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1517 "Python/bytecodes.c" + #line 1518 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2201,9 +2202,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2205 "Python/generated_cases.c.h" + #line 2206 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1551 "Python/bytecodes.c" + #line 1552 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2212,12 +2213,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2216 "Python/generated_cases.c.h" + #line 2217 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1560 "Python/bytecodes.c" + #line 1561 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2221 "Python/generated_cases.c.h" + #line 2222 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2231,7 +2232,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1565 "Python/bytecodes.c" + #line 1566 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2244,7 +2245,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2248 "Python/generated_cases.c.h" + #line 2249 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2259,7 +2260,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1581 "Python/bytecodes.c" + #line 1582 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2272,7 +2273,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2276 "Python/generated_cases.c.h" + #line 2277 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2287,7 +2288,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1597 "Python/bytecodes.c" + #line 1598 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2314,7 +2315,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2318 "Python/generated_cases.c.h" + #line 2319 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2329,7 +2330,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1627 "Python/bytecodes.c" + #line 1628 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2339,7 +2340,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2343 "Python/generated_cases.c.h" + #line 2344 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2354,7 +2355,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1640 "Python/bytecodes.c" + #line 1641 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2366,7 +2367,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2370 "Python/generated_cases.c.h" + #line 2371 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2380,7 +2381,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1655 "Python/bytecodes.c" + #line 1656 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2403,7 +2404,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2407 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2411,7 +2412,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1680 "Python/bytecodes.c" + #line 1681 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2436,7 +2437,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2440 "Python/generated_cases.c.h" + #line 2441 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2444,7 +2445,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1707 "Python/bytecodes.c" + #line 1708 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2462,7 +2463,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2466 "Python/generated_cases.c.h" + #line 2467 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2473,7 +2474,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1727 "Python/bytecodes.c" + #line 1728 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2512,7 +2513,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2516 "Python/generated_cases.c.h" + #line 2517 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2523,7 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1768 "Python/bytecodes.c" + #line 1769 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2533,7 +2534,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2537 "Python/generated_cases.c.h" + #line 2538 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2545,7 +2546,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1787 "Python/bytecodes.c" + #line 1788 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2558,12 +2559,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2562 "Python/generated_cases.c.h" + #line 2563 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1800 "Python/bytecodes.c" + #line 1801 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2567 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2574,7 +2575,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1804 "Python/bytecodes.c" + #line 1805 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2586,7 +2587,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2590 "Python/generated_cases.c.h" + #line 2591 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2597,7 +2598,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1819 "Python/bytecodes.c" + #line 1820 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2613,7 +2614,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2617 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2624,7 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1838 "Python/bytecodes.c" + #line 1839 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2637,7 +2638,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2641 "Python/generated_cases.c.h" + #line 2642 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2648,14 +2649,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1853 "Python/bytecodes.c" + #line 1854 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2654 "Python/generated_cases.c.h" + #line 2655 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1855 "Python/bytecodes.c" + #line 1856 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2659 "Python/generated_cases.c.h" + #line 2660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2665,15 +2666,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1859 "Python/bytecodes.c" + #line 1860 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2671 "Python/generated_cases.c.h" + #line 2672 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1861 "Python/bytecodes.c" + #line 1862 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2677 "Python/generated_cases.c.h" + #line 2678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2684,12 +2685,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1866 "Python/bytecodes.c" + #line 1867 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2690 "Python/generated_cases.c.h" + #line 2691 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1868 "Python/bytecodes.c" + #line 1869 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2697,10 +2698,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2701 "Python/generated_cases.c.h" + #line 2702 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1876 "Python/bytecodes.c" + #line 1877 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2709,7 +2710,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2713 "Python/generated_cases.c.h" + #line 2714 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2719,21 +2720,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1887 "Python/bytecodes.c" + #line 1888 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2726 "Python/generated_cases.c.h" + #line 2727 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1890 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2733 "Python/generated_cases.c.h" + #line 2734 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1895 "Python/bytecodes.c" + #line 1896 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2737 "Python/generated_cases.c.h" + #line 2738 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2742,15 +2743,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1899 "Python/bytecodes.c" + #line 1900 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2749 "Python/generated_cases.c.h" + #line 2750 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1902 "Python/bytecodes.c" + #line 1903 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2754 "Python/generated_cases.c.h" + #line 2755 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2759,29 +2760,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1906 "Python/bytecodes.c" + #line 1907 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2767 "Python/generated_cases.c.h" + #line 2768 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1912 "Python/bytecodes.c" + #line 1913 "Python/bytecodes.c" JUMPBY(oparg); - #line 2776 "Python/generated_cases.c.h" + #line 2777 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1916 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2785 "Python/generated_cases.c.h" + #line 2786 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2789,7 +2790,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1922 "Python/bytecodes.c" + #line 1923 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2799,9 +2800,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2803 "Python/generated_cases.c.h" + #line 2804 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1932 "Python/bytecodes.c" + #line 1933 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2809,14 +2810,14 @@ if (err < 0) goto pop_1_error; } } - #line 2813 "Python/generated_cases.c.h" + #line 2814 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1942 "Python/bytecodes.c" + #line 1943 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2826,9 +2827,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2830 "Python/generated_cases.c.h" + #line 2831 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1952 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2836,67 +2837,67 @@ if (err < 0) goto pop_1_error; } } - #line 2840 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1962 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2849 "Python/generated_cases.c.h" + #line 2850 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1964 "Python/bytecodes.c" + #line 1965 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2857 "Python/generated_cases.c.h" + #line 2858 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1972 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2870 "Python/generated_cases.c.h" + #line 2871 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1978 "Python/bytecodes.c" + #line 1979 "Python/bytecodes.c" } - #line 2874 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1982 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2887 "Python/generated_cases.c.h" + #line 2888 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1991 "Python/bytecodes.c" + #line 1992 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2900 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2907,16 +2908,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1999 "Python/bytecodes.c" + #line 2000 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2916 "Python/generated_cases.c.h" + #line 2917 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2004 "Python/bytecodes.c" + #line 2005 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2924,7 +2925,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2928 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2933,10 +2934,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2014 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2940 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2946,10 +2947,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2020 "Python/bytecodes.c" + #line 2021 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2953 "Python/generated_cases.c.h" + #line 2954 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2960,11 +2961,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2026 "Python/bytecodes.c" + #line 2027 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2968 "Python/generated_cases.c.h" + #line 2969 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2973,14 +2974,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2032 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2980 "Python/generated_cases.c.h" + #line 2981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2035 "Python/bytecodes.c" + #line 2036 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2984 "Python/generated_cases.c.h" + #line 2985 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2988,7 +2989,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2039 "Python/bytecodes.c" + #line 2040 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3011,11 +3012,11 @@ if (iter == NULL) { goto error; } - #line 3015 "Python/generated_cases.c.h" + #line 3016 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2062 "Python/bytecodes.c" + #line 2063 "Python/bytecodes.c" } - #line 3019 "Python/generated_cases.c.h" + #line 3020 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3026,7 +3027,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2081 "Python/bytecodes.c" + #line 2082 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3057,7 +3058,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3061 "Python/generated_cases.c.h" + #line 3062 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3065,7 +3066,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2114 "Python/bytecodes.c" + #line 2115 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3091,14 +3092,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3095 "Python/generated_cases.c.h" + #line 3096 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2142 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3118,7 +3119,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3122 "Python/generated_cases.c.h" + #line 3123 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3128,7 +3129,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2164 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3148,7 +3149,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3152 "Python/generated_cases.c.h" + #line 3153 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3158,7 +3159,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2186 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3176,7 +3177,7 @@ if (next == NULL) { goto error; } - #line 3180 "Python/generated_cases.c.h" + #line 3181 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3185,7 +3186,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2206 "Python/bytecodes.c" + #line 2207 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3200,14 +3201,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3204 "Python/generated_cases.c.h" + #line 3205 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2223 "Python/bytecodes.c" + #line 2224 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3230,16 +3231,16 @@ Py_DECREF(enter); goto error; } - #line 3234 "Python/generated_cases.c.h" + #line 3235 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2246 "Python/bytecodes.c" + #line 2247 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3243 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3251,7 +3252,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2256 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3277,16 +3278,16 @@ Py_DECREF(enter); goto error; } - #line 3281 "Python/generated_cases.c.h" + #line 3282 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2282 "Python/bytecodes.c" + #line 2283 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3290 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3298,7 +3299,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2291 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3319,7 +3320,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3323 "Python/generated_cases.c.h" + #line 3324 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3328,7 +3329,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2314 "Python/bytecodes.c" + #line 2315 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3338,7 +3339,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3342 "Python/generated_cases.c.h" + #line 3343 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3352,7 +3353,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2326 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3369,7 +3370,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3373 "Python/generated_cases.c.h" + #line 3374 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3383,7 +3384,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2345 "Python/bytecodes.c" + #line 2346 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3393,7 +3394,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3397 "Python/generated_cases.c.h" + #line 3398 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3407,7 +3408,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2357 "Python/bytecodes.c" + #line 2358 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3421,7 +3422,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3426 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3430,16 +3431,16 @@ } TARGET(KW_NAMES) { - #line 2373 "Python/bytecodes.c" + #line 2374 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3438 "Python/generated_cases.c.h" + #line 3439 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2379 "Python/bytecodes.c" + #line 2380 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3452,7 +3453,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3456 "Python/generated_cases.c.h" + #line 3457 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3462,7 +3463,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2424 "Python/bytecodes.c" + #line 2425 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3543,7 +3544,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3547 "Python/generated_cases.c.h" + #line 3548 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3555,7 +3556,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2511 "Python/bytecodes.c" + #line 2512 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3565,7 +3566,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3569 "Python/generated_cases.c.h" + #line 3570 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3574,7 +3575,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2523 "Python/bytecodes.c" + #line 2524 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3599,7 +3600,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3603 "Python/generated_cases.c.h" + #line 3604 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3608,7 +3609,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2550 "Python/bytecodes.c" + #line 2551 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3638,7 +3639,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3642 "Python/generated_cases.c.h" + #line 3643 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3646,7 +3647,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2582 "Python/bytecodes.c" + #line 2583 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3656,7 +3657,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3660 "Python/generated_cases.c.h" + #line 3661 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3669,7 +3670,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2594 "Python/bytecodes.c" + #line 2595 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3680,7 +3681,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3684 "Python/generated_cases.c.h" + #line 3685 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3694,7 +3695,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2608 "Python/bytecodes.c" + #line 2609 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3705,7 +3706,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3709 "Python/generated_cases.c.h" + #line 3710 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3719,7 +3720,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2622 "Python/bytecodes.c" + #line 2623 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3741,7 +3742,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3745 "Python/generated_cases.c.h" + #line 3746 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3755,7 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2647 "Python/bytecodes.c" + #line 2648 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3783,7 +3784,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3787 "Python/generated_cases.c.h" + #line 3788 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3797,7 +3798,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2678 "Python/bytecodes.c" + #line 2679 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3829,7 +3830,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3833 "Python/generated_cases.c.h" + #line 3834 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3843,7 +3844,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2713 "Python/bytecodes.c" + #line 2714 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3875,7 +3876,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3879 "Python/generated_cases.c.h" + #line 3880 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3889,7 +3890,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2748 "Python/bytecodes.c" + #line 2749 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3914,7 +3915,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3918 "Python/generated_cases.c.h" + #line 3919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3927,7 +3928,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2775 "Python/bytecodes.c" + #line 2776 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3954,7 +3955,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3958 "Python/generated_cases.c.h" + #line 3959 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3966,7 +3967,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2805 "Python/bytecodes.c" + #line 2806 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3984,14 +3985,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3988 "Python/generated_cases.c.h" + #line 3989 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2825 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4022,7 +4023,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4026 "Python/generated_cases.c.h" + #line 4027 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4035,7 +4036,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2859 "Python/bytecodes.c" + #line 2860 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4064,7 +4065,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4068 "Python/generated_cases.c.h" + #line 4069 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4077,7 +4078,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2891 "Python/bytecodes.c" + #line 2892 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4106,7 +4107,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4110 "Python/generated_cases.c.h" + #line 4111 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4119,7 +4120,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2923 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4147,7 +4148,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4151 "Python/generated_cases.c.h" + #line 4152 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4157,9 +4158,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2954 "Python/bytecodes.c" + #line 2955 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4163 "Python/generated_cases.c.h" + #line 4164 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4168,7 +4169,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2958 "Python/bytecodes.c" + #line 2959 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4211,14 +4212,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4215 "Python/generated_cases.c.h" + #line 4216 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3001 "Python/bytecodes.c" + #line 3002 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4222 "Python/generated_cases.c.h" + #line 4223 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4233,7 +4234,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3011 "Python/bytecodes.c" + #line 3012 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4262,14 +4263,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4266 "Python/generated_cases.c.h" + #line 4267 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3042 "Python/bytecodes.c" + #line 3043 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4290,7 +4291,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4294 "Python/generated_cases.c.h" + #line 4295 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4298,15 +4299,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3065 "Python/bytecodes.c" + #line 3066 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4304 "Python/generated_cases.c.h" + #line 4305 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3067 "Python/bytecodes.c" + #line 3068 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4310 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4317,7 +4318,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3071 "Python/bytecodes.c" + #line 3072 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4352,7 +4353,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4356 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4361,10 +4362,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3108 "Python/bytecodes.c" + #line 3109 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4376,7 +4377,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3113 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4391,12 +4392,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4395 "Python/generated_cases.c.h" + #line 4396 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3128 "Python/bytecodes.c" + #line 3129 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4400 "Python/generated_cases.c.h" + #line 4401 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4406,21 +4407,22 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3133 "Python/bytecodes.c" + #line 3134 "Python/bytecodes.c" assert(oparg >= 2); - #line 4412 "Python/generated_cases.c.h" + #line 4413 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3137 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (original_opcode < 0) { next_instr = here+1; goto error; @@ -4435,11 +4437,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4439 "Python/generated_cases.c.h" + #line 4441 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3159 "Python/bytecodes.c" + #line 3161 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4451,26 +4453,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4455 "Python/generated_cases.c.h" + #line 4457 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3174 "Python/bytecodes.c" + #line 3176 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4461 "Python/generated_cases.c.h" + #line 4463 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3179 "Python/bytecodes.c" + #line 3181 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4468 "Python/generated_cases.c.h" + #line 4470 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3184 "Python/bytecodes.c" + #line 3186 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4479,12 +4481,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4483 "Python/generated_cases.c.h" + #line 4485 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3195 "Python/bytecodes.c" + #line 3197 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4493,12 +4495,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4497 "Python/generated_cases.c.h" + #line 4499 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3206 "Python/bytecodes.c" + #line 3208 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4511,12 +4513,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4515 "Python/generated_cases.c.h" + #line 4517 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3221 "Python/bytecodes.c" + #line 3223 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4529,22 +4531,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4535 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3236 "Python/bytecodes.c" + #line 3238 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4544 "Python/generated_cases.c.h" + #line 4546 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3244 "Python/bytecodes.c" + #line 3246 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4550 "Python/generated_cases.c.h" + #line 4552 "Python/generated_cases.c.h" } From 6c3473ab9b21b22338033c63068eeaf7b003a8a2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:41:59 +0000 Subject: [PATCH 065/116] Minor cleanups. --- Python/instrumentation.c | 65 +++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 76cb031f27c198..e2c54c32185e50 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,6 +1,7 @@ #include "Python.h" +#include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_namespace.h" @@ -857,7 +858,6 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); assert(tools_is_subset_for_event(code, event, tools)); - assert(_Py_popcount32(tools) == 1); } instrument(code, offset); } @@ -934,7 +934,7 @@ call_one_instrument( int old_what = tstate->what_event; tstate->what_event = event; tstate->tracing++; - PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL); tstate->tracing--; tstate->what_event = old_what; if (res == NULL) { @@ -1005,40 +1005,13 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) } static int -call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, int offset, uint8_t tools) +call_instrumentation_vector( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) { - //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } - PyInterpreterState *interp = tstate->interp; - while (tools) { - int tool = most_significant_bit(tools); - assert(tool >= 0 && tool < 8); - assert(tools & (1 << tool)); - tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); - if (res == 0) { - /* Nothing to do */ - } - else if (res < 0) { - /* error */ - return -1; - } - else { - /* DISABLE */ - remove_tools(code, offset, event, 1 << tool); - } - } - return 0; -} - -static int -call_instrumentation_vector( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) -{ assert(!_PyErr_Occurred(tstate)); assert(args[0] == NULL); PyCodeObject *code = frame->f_code; @@ -1055,7 +1028,29 @@ call_instrumentation_vector( assert(args[2] == NULL); args[2] = offset_obj; uint8_t tools = get_tools_for_instruction(code, offset, event); - int err = call_instrument(tstate, code, event, &args[1], nargsf | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + PyInterpreterState *interp = tstate->interp; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + remove_tools(code, offset, event, 1 << tool); + } + } Py_DECREF(offset_obj); return err; } @@ -1111,11 +1106,11 @@ _Py_call_instrumentation_jump( static void call_instrumentation_vector_protected( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); PyObject *exc = _PyErr_GetRaisedException(tstate); - int err = call_instrumentation_vector(tstate, event, frame, instr, nargsf, args); + int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args); if (err) { Py_XDECREF(exc); } From 899aecda675eb0041d05720204a9b163522b32af Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 06:17:25 +0000 Subject: [PATCH 066/116] Remove useless asserts. --- Python/legacy_tracing.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index b55a93ab64ab4f..c72af86183bc29 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -196,7 +196,6 @@ sys_trace_instruction_func( assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -253,7 +252,6 @@ sys_trace_line_func( int line = _PyLong_AsInt(args[1]); assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -284,7 +282,6 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); From 80d2e2e37e5aa6e894f81798a7b5d8990af98176 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 07:48:48 +0000 Subject: [PATCH 067/116] Make functions static --- Python/instrumentation.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e2c54c32185e50..5b6314021961f9 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -301,7 +301,8 @@ typedef struct _Instruction { bool is_specialized_instrumented; } Instruction; -Instruction read_instruction(PyCodeObject *code, int offset) +static Instruction +read_instruction(PyCodeObject *code, int offset) { Instruction result = (Instruction){0}; int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; @@ -812,7 +813,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } #ifndef NDEBUG -bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) +static bool +tools_is_subset_for_event(PyCodeObject * code, int event, int tools) { int global_tools = PyInterpreterState_Get()->monitors.tools[event]; int local_tools = code->_co_monitoring->local_monitors.tools[event]; From 94d35d829d22775590118f3e5dd4f8b1f9ef6827 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 08:52:53 +0000 Subject: [PATCH 068/116] Update Windows build files. --- PCbuild/_freeze_module.vcxproj | 2 ++ PCbuild/_freeze_module.vcxproj.filters | 6 ++++++ PCbuild/pythoncore.vcxproj | 2 ++ PCbuild/pythoncore.vcxproj.filters | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 4f39756019e692..9f7bb27ba2a98b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -207,6 +207,8 @@ + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7d7c4587b9a3f3..77dc9ef7215a02 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -208,6 +208,12 @@ Source Files + + Source Files + + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c754b2165745ff..54c125f34c7697 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -528,6 +528,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 90ed0602821bff..3b07728e0abc65 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1169,6 +1169,12 @@ Source Files + + Source Files + + + Source Files + Python From 5aa080575ba92e6a732a13e7a52de4a8f6515932 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 09:11:18 +0000 Subject: [PATCH 069/116] Make _PyLegacyEventHandler_Type immortal --- Python/legacy_tracing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index c72af86183bc29..514afc00f7adf0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -7,6 +7,7 @@ #include "opcode.h" #include "pycore_ceval.h" #include "pycore_instruments.h" +#include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pymem.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -336,7 +337,7 @@ sys_trace_exception_handled( PyTypeObject _PyLegacyEventHandler_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), .tp_dealloc = (destructor)dealloc, From b39edd311ea0e5bf2963a4b307480e6b6cc47d4d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 09:19:31 +0000 Subject: [PATCH 070/116] Make arrays const. --- Python/instrumentation.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5b6314021961f9..0147b4c35f1647 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1479,7 +1479,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -static bool super_instructions[256] = { +static const bool super_instructions[256] = { [LOAD_FAST__LOAD_FAST] = true, [LOAD_FAST__LOAD_CONST] = true, [STORE_FAST__LOAD_FAST] = true, @@ -1972,7 +1972,7 @@ add_power2_constant(PyObject *obj, const char *name, int i) return err; } -const char *event_names[] = { +const char *const event_names [] = { [PY_MONITORING_EVENT_PY_START] = "PY_START", [PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME", [PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN", From c9c40cbd393ed415365aa4c4938942186f26af5e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 15:26:03 +0000 Subject: [PATCH 071/116] Rename _PyFrame_GetStackPointer to _PyFrame_FetchStackPointer and make it responsible for setting stacktop to -1. --- Include/internal/pycore_frame.h | 10 +- Python/bytecodes.c | 6 +- Python/ceval.c | 27 +- Python/generated_cases.c.h | 968 ++++++++++++++++---------------- 4 files changed, 495 insertions(+), 516 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8af4f13527ecb7..93b37e6fee64b3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -138,10 +138,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } +/* Fetches the stack pointer, and sets stacktop to -1. + Having stacktop <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. */ static inline PyObject** -_PyFrame_GetStackPointer(_PyInterpreterFrame *frame) +_PyFrame_FetchStackPointer(_PyInterpreterFrame *frame) { - return frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus+frame->stacktop; + frame->stacktop = -1; + return sp; } static inline void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fd213f8e7282e7..f8df0ccb030889 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -99,8 +99,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -3139,8 +3138,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/ceval.c b/Python/ceval.c index 004410fe4dc1e6..c2b0dd5ea95776 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -214,20 +214,6 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" -#define UPDATE_NEXT_OR_JUMP(offset) \ -do { \ - assert(frame->stacktop >= 0); \ - if (frame->prev_instr != next_instr-1) { \ - next_instr = frame->prev_instr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - } \ - else { \ - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); \ - JUMPBY(offset); \ - } \ - frame->stacktop = -1; \ -} while (0) - #ifdef HAVE_ERRNO_H #include #endif @@ -631,8 +617,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - frame->stacktop = -1; \ + stack_pointer = _PyFrame_FetchStackPointer(frame); \ if (err) { \ next_instr = (dest)+1; \ goto error; \ @@ -738,15 +723,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - /* Set stackdepth to -1. \ - Update when returning or calling trace function. \ - Having stackdepth <= 0 ensures that invalid \ - values are not visible to the cycle GC. \ - We choose -1 rather than 0 to assist debugging. \ - */ \ - frame->stacktop = -1; - + stack_pointer = _PyFrame_FetchStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6531f4fd764ca6..08a294e4803e9e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,8 +21,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -33,12 +32,12 @@ goto handle_eval_breaker; } } - #line 37 "Python/generated_cases.c.h" + #line 36 "Python/generated_cases.c.h" DISPATCH(); } TARGET(RESUME) { - #line 117 "Python/bytecodes.c" + #line 116 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -50,18 +49,18 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } - #line 54 "Python/generated_cases.c.h" + #line 53 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 131 "Python/bytecodes.c" + #line 130 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 65 "Python/generated_cases.c.h" + #line 64 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -69,11 +68,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 138 "Python/bytecodes.c" + #line 137 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 77 "Python/generated_cases.c.h" + #line 76 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -81,11 +80,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 89 "Python/generated_cases.c.h" + #line 88 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -94,10 +93,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 101 "Python/generated_cases.c.h" + #line 100 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -105,9 +104,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 111 "Python/generated_cases.c.h" + #line 110 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -117,21 +116,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 125 "Python/generated_cases.c.h" + #line 124 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 135 "Python/generated_cases.c.h" + #line 134 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -145,20 +144,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 153 "Python/generated_cases.c.h" + #line 152 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 162 "Python/generated_cases.c.h" + #line 161 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -171,18 +170,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 177 "Python/generated_cases.c.h" + #line 176 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 186 "Python/generated_cases.c.h" + #line 185 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -194,16 +193,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 200 "Python/generated_cases.c.h" + #line 199 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 207 "Python/generated_cases.c.h" + #line 206 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -214,20 +213,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 221 "Python/generated_cases.c.h" + #line 220 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 231 "Python/generated_cases.c.h" + #line 230 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -238,8 +237,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 165 "Python/bytecodes.c" - #line 243 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 242 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -247,9 +246,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 169 "Python/bytecodes.c" + #line 168 "Python/bytecodes.c" res = NULL; - #line 253 "Python/generated_cases.c.h" + #line 252 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -260,14 +259,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 165 "Python/bytecodes.c" - #line 265 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 264 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 165 "Python/bytecodes.c" - #line 271 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 270 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -277,7 +276,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 175 "Python/bytecodes.c" + #line 174 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -287,7 +286,7 @@ } PyErr_SetRaisedException(NULL); } - #line 291 "Python/generated_cases.c.h" + #line 290 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -297,9 +296,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 188 "Python/bytecodes.c" + #line 187 "Python/bytecodes.c" Py_DECREF(receiver); - #line 303 "Python/generated_cases.c.h" + #line 302 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -308,7 +307,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 192 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -317,7 +316,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 321 "Python/generated_cases.c.h" + #line 320 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -326,13 +325,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 203 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 332 "Python/generated_cases.c.h" + #line 331 "Python/generated_cases.c.h" Py_DECREF(value); - #line 205 "Python/bytecodes.c" + #line 204 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 336 "Python/generated_cases.c.h" + #line 335 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -340,11 +339,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 209 "Python/bytecodes.c" + #line 208 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 346 "Python/generated_cases.c.h" + #line 345 "Python/generated_cases.c.h" Py_DECREF(value); - #line 211 "Python/bytecodes.c" + #line 210 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -353,7 +352,7 @@ res = Py_False; } Py_INCREF(res); - #line 357 "Python/generated_cases.c.h" + #line 356 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -361,13 +360,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 222 "Python/bytecodes.c" + #line 221 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 367 "Python/generated_cases.c.h" + #line 366 "Python/generated_cases.c.h" Py_DECREF(value); - #line 224 "Python/bytecodes.c" + #line 223 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 371 "Python/generated_cases.c.h" + #line 370 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -376,7 +375,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 241 "Python/bytecodes.c" + #line 240 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -384,7 +383,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 388 "Python/generated_cases.c.h" + #line 387 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -395,14 +394,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 251 "Python/bytecodes.c" + #line 250 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 406 "Python/generated_cases.c.h" + #line 405 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -413,7 +412,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 260 "Python/bytecodes.c" + #line 259 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -421,7 +420,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 425 "Python/generated_cases.c.h" + #line 424 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -432,13 +431,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 270 "Python/bytecodes.c" + #line 269 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 442 "Python/generated_cases.c.h" + #line 441 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -449,7 +448,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 278 "Python/bytecodes.c" + #line 277 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -457,7 +456,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 461 "Python/generated_cases.c.h" + #line 460 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -467,7 +466,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 294 "Python/bytecodes.c" + #line 293 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -494,7 +493,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 498 "Python/generated_cases.c.h" + #line 497 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -503,14 +502,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 323 "Python/bytecodes.c" + #line 322 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 514 "Python/generated_cases.c.h" + #line 513 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -521,7 +520,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 332 "Python/bytecodes.c" + #line 331 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -529,7 +528,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 533 "Python/generated_cases.c.h" + #line 532 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -542,7 +541,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 350 "Python/bytecodes.c" + #line 349 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -554,12 +553,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 558 "Python/generated_cases.c.h" + #line 557 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 362 "Python/bytecodes.c" + #line 361 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 563 "Python/generated_cases.c.h" + #line 562 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -571,7 +570,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 366 "Python/bytecodes.c" + #line 365 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -584,7 +583,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 588 "Python/generated_cases.c.h" + #line 587 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -595,7 +594,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 381 "Python/bytecodes.c" + #line 380 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -608,7 +607,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 612 "Python/generated_cases.c.h" + #line 611 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -617,7 +616,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 396 "Python/bytecodes.c" + #line 395 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -631,7 +630,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 635 "Python/generated_cases.c.h" + #line 634 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -642,7 +641,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 412 "Python/bytecodes.c" + #line 411 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -656,7 +655,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 660 "Python/generated_cases.c.h" + #line 659 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -667,7 +666,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 428 "Python/bytecodes.c" + #line 427 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -675,14 +674,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 679 "Python/generated_cases.c.h" + #line 678 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 436 "Python/bytecodes.c" + #line 435 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 686 "Python/generated_cases.c.h" + #line 685 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -696,7 +695,7 @@ PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); - #line 443 "Python/bytecodes.c" + #line 442 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -715,15 +714,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 719 "Python/generated_cases.c.h" + #line 718 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 464 "Python/bytecodes.c" + #line 463 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 727 "Python/generated_cases.c.h" + #line 726 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -732,13 +731,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 469 "Python/bytecodes.c" + #line 468 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 738 "Python/generated_cases.c.h" + #line 737 "Python/generated_cases.c.h" Py_DECREF(v); - #line 471 "Python/bytecodes.c" + #line 470 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 742 "Python/generated_cases.c.h" + #line 741 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -751,7 +750,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 482 "Python/bytecodes.c" + #line 481 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -766,13 +765,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 770 "Python/generated_cases.c.h" + #line 769 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 497 "Python/bytecodes.c" + #line 496 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 776 "Python/generated_cases.c.h" + #line 775 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -782,7 +781,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 501 "Python/bytecodes.c" + #line 500 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -799,7 +798,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 803 "Python/generated_cases.c.h" + #line 802 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -809,13 +808,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 520 "Python/bytecodes.c" + #line 519 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 819 "Python/generated_cases.c.h" + #line 818 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -824,15 +823,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 528 "Python/bytecodes.c" + #line 527 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 831 "Python/generated_cases.c.h" + #line 830 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 531 "Python/bytecodes.c" + #line 530 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 836 "Python/generated_cases.c.h" + #line 835 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -840,14 +839,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 535 "Python/bytecodes.c" + #line 534 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 847 "Python/generated_cases.c.h" + #line 846 "Python/generated_cases.c.h" Py_DECREF(value); - #line 538 "Python/bytecodes.c" + #line 537 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 851 "Python/generated_cases.c.h" + #line 850 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -856,15 +855,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 542 "Python/bytecodes.c" + #line 541 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 863 "Python/generated_cases.c.h" + #line 862 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 545 "Python/bytecodes.c" + #line 544 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 868 "Python/generated_cases.c.h" + #line 867 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -872,7 +871,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 549 "Python/bytecodes.c" + #line 548 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -890,12 +889,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 894 "Python/generated_cases.c.h" + #line 893 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 569 "Python/bytecodes.c" + #line 568 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -906,12 +905,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 910 "Python/generated_cases.c.h" + #line 909 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 582 "Python/bytecodes.c" + #line 581 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -923,12 +922,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 927 "Python/generated_cases.c.h" + #line 926 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 596 "Python/bytecodes.c" + #line 595 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -944,11 +943,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 948 "Python/generated_cases.c.h" + #line 947 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 614 "Python/bytecodes.c" + #line 613 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -961,11 +960,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 965 "Python/generated_cases.c.h" + #line 964 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 629 "Python/bytecodes.c" + #line 628 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); int err = _Py_call_instrumentation_arg( @@ -982,13 +981,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 986 "Python/generated_cases.c.h" + #line 985 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 648 "Python/bytecodes.c" + #line 647 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1001,16 +1000,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1005 "Python/generated_cases.c.h" + #line 1004 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 661 "Python/bytecodes.c" + #line 660 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1012 "Python/generated_cases.c.h" + #line 1011 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 666 "Python/bytecodes.c" + #line 665 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1023,7 +1022,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1027 "Python/generated_cases.c.h" + #line 1026 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1031,7 +1030,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 681 "Python/bytecodes.c" + #line 680 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1075,7 +1074,7 @@ } } - #line 1079 "Python/generated_cases.c.h" + #line 1078 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1086,16 +1085,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 728 "Python/bytecodes.c" + #line 727 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1097 "Python/generated_cases.c.h" + #line 1096 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 735 "Python/bytecodes.c" + #line 734 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1113,7 +1112,7 @@ if (iter == NULL) goto pop_1_error; - #line 1117 "Python/generated_cases.c.h" + #line 1116 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1124,7 +1123,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 761 "Python/bytecodes.c" + #line 760 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1170,7 +1169,7 @@ } } Py_DECREF(v); - #line 1174 "Python/generated_cases.c.h" + #line 1173 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1179,7 +1178,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 809 "Python/bytecodes.c" + #line 808 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1194,12 +1193,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1198 "Python/generated_cases.c.h" + #line 1197 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 826 "Python/bytecodes.c" + #line 825 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1217,12 +1216,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1221 "Python/generated_cases.c.h" + #line 1220 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 846 "Python/bytecodes.c" + #line 845 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1239,15 +1238,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1243 "Python/generated_cases.c.h" + #line 1242 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 865 "Python/bytecodes.c" + #line 864 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1251 "Python/generated_cases.c.h" + #line 1250 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1255,7 +1254,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 870 "Python/bytecodes.c" + #line 869 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1273,26 +1272,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1277 "Python/generated_cases.c.h" + #line 1276 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 890 "Python/bytecodes.c" + #line 889 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1286 "Python/generated_cases.c.h" + #line 1285 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 893 "Python/bytecodes.c" + #line 892 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1296 "Python/generated_cases.c.h" + #line 1295 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1303,23 +1302,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 902 "Python/bytecodes.c" + #line 901 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1312 "Python/generated_cases.c.h" + #line 1311 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 907 "Python/bytecodes.c" + #line 906 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1323 "Python/generated_cases.c.h" + #line 1322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1328,9 +1327,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 916 "Python/bytecodes.c" + #line 915 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1334 "Python/generated_cases.c.h" + #line 1333 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1338,7 +1337,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 920 "Python/bytecodes.c" + #line 919 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1360,7 +1359,7 @@ if (true) goto error; } } - #line 1364 "Python/generated_cases.c.h" + #line 1363 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1368,33 +1367,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 944 "Python/bytecodes.c" + #line 943 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1379 "Python/generated_cases.c.h" + #line 1378 "Python/generated_cases.c.h" Py_DECREF(v); - #line 951 "Python/bytecodes.c" + #line 950 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1388 "Python/generated_cases.c.h" + #line 1387 "Python/generated_cases.c.h" Py_DECREF(v); - #line 958 "Python/bytecodes.c" + #line 957 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1392 "Python/generated_cases.c.h" + #line 1391 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 962 "Python/bytecodes.c" + #line 961 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1411,7 +1410,7 @@ name); goto error; } - #line 1415 "Python/generated_cases.c.h" + #line 1414 "Python/generated_cases.c.h" DISPATCH(); } @@ -1419,7 +1418,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 988 "Python/bytecodes.c" + #line 987 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1432,11 +1431,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1436 "Python/generated_cases.c.h" + #line 1435 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1001 "Python/bytecodes.c" + #line 1000 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1440 "Python/generated_cases.c.h" + #line 1439 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1446,14 +1445,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1005 "Python/bytecodes.c" + #line 1004 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1457 "Python/generated_cases.c.h" + #line 1456 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1464,7 +1463,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1015 "Python/bytecodes.c" + #line 1014 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1472,7 +1471,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1476 "Python/generated_cases.c.h" + #line 1475 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1483,7 +1482,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1026 "Python/bytecodes.c" + #line 1025 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1491,7 +1490,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1495 "Python/generated_cases.c.h" + #line 1494 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1501,15 +1500,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1037 "Python/bytecodes.c" + #line 1036 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1509 "Python/generated_cases.c.h" + #line 1508 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1041 "Python/bytecodes.c" + #line 1040 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1513 "Python/generated_cases.c.h" + #line 1512 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1520,7 +1519,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1052 "Python/bytecodes.c" + #line 1051 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1536,12 +1535,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1540 "Python/generated_cases.c.h" + #line 1539 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1068 "Python/bytecodes.c" + #line 1067 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1545 "Python/generated_cases.c.h" + #line 1544 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1549,34 +1548,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1072 "Python/bytecodes.c" + #line 1071 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1556 "Python/generated_cases.c.h" + #line 1555 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1075 "Python/bytecodes.c" + #line 1074 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1560 "Python/generated_cases.c.h" + #line 1559 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1079 "Python/bytecodes.c" + #line 1078 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1570 "Python/generated_cases.c.h" + #line 1569 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1082 "Python/bytecodes.c" + #line 1081 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1574 "Python/generated_cases.c.h" + #line 1573 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1086 "Python/bytecodes.c" + #line 1085 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1588,13 +1587,13 @@ } goto error; } - #line 1592 "Python/generated_cases.c.h" + #line 1591 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1100 "Python/bytecodes.c" + #line 1099 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1653,7 +1652,7 @@ } } } - #line 1657 "Python/generated_cases.c.h" + #line 1656 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1664,7 +1663,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1167 "Python/bytecodes.c" + #line 1166 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1716,7 +1715,7 @@ } } null = NULL; - #line 1720 "Python/generated_cases.c.h" + #line 1719 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1730,7 +1729,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1221 "Python/bytecodes.c" + #line 1220 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1741,7 +1740,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1745 "Python/generated_cases.c.h" + #line 1744 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1756,7 +1755,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1234 "Python/bytecodes.c" + #line 1233 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1770,7 +1769,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1774 "Python/generated_cases.c.h" + #line 1773 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1780,16 +1779,16 @@ } TARGET(DELETE_FAST) { - #line 1250 "Python/bytecodes.c" + #line 1249 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1788 "Python/generated_cases.c.h" + #line 1787 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1256 "Python/bytecodes.c" + #line 1255 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1798,12 +1797,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1802 "Python/generated_cases.c.h" + #line 1801 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1267 "Python/bytecodes.c" + #line 1266 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1814,13 +1813,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1818 "Python/generated_cases.c.h" + #line 1817 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1280 "Python/bytecodes.c" + #line 1279 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1852,7 +1851,7 @@ } Py_INCREF(value); } - #line 1856 "Python/generated_cases.c.h" + #line 1855 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1860,7 +1859,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1314 "Python/bytecodes.c" + #line 1313 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1868,7 +1867,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1872 "Python/generated_cases.c.h" + #line 1871 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1876,18 +1875,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1324 "Python/bytecodes.c" + #line 1323 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1885 "Python/generated_cases.c.h" + #line 1884 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1331 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1898,22 +1897,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1902 "Python/generated_cases.c.h" + #line 1901 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1344 "Python/bytecodes.c" + #line 1343 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1911 "Python/generated_cases.c.h" + #line 1910 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1346 "Python/bytecodes.c" + #line 1345 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1917 "Python/generated_cases.c.h" + #line 1916 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1923,10 +1922,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1350 "Python/bytecodes.c" + #line 1349 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1930 "Python/generated_cases.c.h" + #line 1929 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1936,10 +1935,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1355 "Python/bytecodes.c" + #line 1354 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1943 "Python/generated_cases.c.h" + #line 1942 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1949,7 +1948,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1360 "Python/bytecodes.c" + #line 1359 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1960,13 +1959,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1964 "Python/generated_cases.c.h" + #line 1963 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1371 "Python/bytecodes.c" + #line 1370 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1970 "Python/generated_cases.c.h" + #line 1969 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1975,13 +1974,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1378 "Python/bytecodes.c" + #line 1377 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1981 "Python/generated_cases.c.h" + #line 1980 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1380 "Python/bytecodes.c" + #line 1379 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1985 "Python/generated_cases.c.h" + #line 1984 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1989,7 +1988,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1384 "Python/bytecodes.c" + #line 1383 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2004,7 +2003,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2008 "Python/generated_cases.c.h" + #line 2007 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2014,7 +2013,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1401 "Python/bytecodes.c" + #line 1400 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2022,13 +2021,13 @@ if (map == NULL) goto error; - #line 2026 "Python/generated_cases.c.h" + #line 2025 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1409 "Python/bytecodes.c" + #line 1408 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2032 "Python/generated_cases.c.h" + #line 2031 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2036,7 +2035,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1413 "Python/bytecodes.c" + #line 1412 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2076,7 +2075,7 @@ Py_DECREF(ann_dict); } } - #line 2080 "Python/generated_cases.c.h" + #line 2079 "Python/generated_cases.c.h" DISPATCH(); } @@ -2084,7 +2083,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1455 "Python/bytecodes.c" + #line 1454 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2094,14 +2093,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2098 "Python/generated_cases.c.h" + #line 2097 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1465 "Python/bytecodes.c" + #line 1464 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2105 "Python/generated_cases.c.h" + #line 2104 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2109,7 +2108,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1469 "Python/bytecodes.c" + #line 1468 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2117,12 +2116,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2121 "Python/generated_cases.c.h" + #line 2120 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1477 "Python/bytecodes.c" + #line 1476 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2126 "Python/generated_cases.c.h" + #line 2125 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2130,17 +2129,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1483 "Python/bytecodes.c" + #line 1482 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2139 "Python/generated_cases.c.h" + #line 2138 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1488 "Python/bytecodes.c" + #line 1487 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2144 "Python/generated_cases.c.h" + #line 2143 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2150,13 +2149,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1495 "Python/bytecodes.c" + #line 1494 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2160 "Python/generated_cases.c.h" + #line 2159 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2168,7 +2167,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1518 "Python/bytecodes.c" + #line 1517 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2202,9 +2201,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2206 "Python/generated_cases.c.h" + #line 2205 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1552 "Python/bytecodes.c" + #line 1551 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2213,12 +2212,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2217 "Python/generated_cases.c.h" + #line 2216 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1561 "Python/bytecodes.c" + #line 1560 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2222 "Python/generated_cases.c.h" + #line 2221 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2232,7 +2231,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1566 "Python/bytecodes.c" + #line 1565 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2245,7 +2244,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2249 "Python/generated_cases.c.h" + #line 2248 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2260,7 +2259,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1582 "Python/bytecodes.c" + #line 1581 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2273,7 +2272,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2277 "Python/generated_cases.c.h" + #line 2276 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2288,7 +2287,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1598 "Python/bytecodes.c" + #line 1597 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2315,7 +2314,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2319 "Python/generated_cases.c.h" + #line 2318 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2330,7 +2329,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1628 "Python/bytecodes.c" + #line 1627 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2340,7 +2339,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2344 "Python/generated_cases.c.h" + #line 2343 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2355,7 +2354,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1641 "Python/bytecodes.c" + #line 1640 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2367,7 +2366,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2371 "Python/generated_cases.c.h" + #line 2370 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2381,7 +2380,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1656 "Python/bytecodes.c" + #line 1655 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2404,7 +2403,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2408 "Python/generated_cases.c.h" + #line 2407 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2412,7 +2411,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1681 "Python/bytecodes.c" + #line 1680 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2437,7 +2436,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2441 "Python/generated_cases.c.h" + #line 2440 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2445,7 +2444,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1708 "Python/bytecodes.c" + #line 1707 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2463,7 +2462,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2467 "Python/generated_cases.c.h" + #line 2466 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2474,7 +2473,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1728 "Python/bytecodes.c" + #line 1727 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2513,7 +2512,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2517 "Python/generated_cases.c.h" + #line 2516 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2524,7 +2523,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1769 "Python/bytecodes.c" + #line 1768 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2534,7 +2533,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2538 "Python/generated_cases.c.h" + #line 2537 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2546,7 +2545,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1788 "Python/bytecodes.c" + #line 1787 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2559,12 +2558,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2563 "Python/generated_cases.c.h" + #line 2562 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1801 "Python/bytecodes.c" + #line 1800 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2568 "Python/generated_cases.c.h" + #line 2567 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2575,7 +2574,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1805 "Python/bytecodes.c" + #line 1804 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2587,7 +2586,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2591 "Python/generated_cases.c.h" + #line 2590 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2598,7 +2597,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1820 "Python/bytecodes.c" + #line 1819 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2614,7 +2613,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2618 "Python/generated_cases.c.h" + #line 2617 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2625,7 +2624,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1839 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2638,7 +2637,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2642 "Python/generated_cases.c.h" + #line 2641 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2649,14 +2648,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1854 "Python/bytecodes.c" + #line 1853 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2655 "Python/generated_cases.c.h" + #line 2654 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1856 "Python/bytecodes.c" + #line 1855 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2660 "Python/generated_cases.c.h" + #line 2659 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2666,15 +2665,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1860 "Python/bytecodes.c" + #line 1859 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2672 "Python/generated_cases.c.h" + #line 2671 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1862 "Python/bytecodes.c" + #line 1861 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2678 "Python/generated_cases.c.h" + #line 2677 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2685,12 +2684,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1867 "Python/bytecodes.c" + #line 1866 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2691 "Python/generated_cases.c.h" + #line 2690 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1869 "Python/bytecodes.c" + #line 1868 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2698,10 +2697,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2702 "Python/generated_cases.c.h" + #line 2701 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1877 "Python/bytecodes.c" + #line 1876 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2710,7 +2709,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2714 "Python/generated_cases.c.h" + #line 2713 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2720,21 +2719,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1888 "Python/bytecodes.c" + #line 1887 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2727 "Python/generated_cases.c.h" + #line 2726 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1891 "Python/bytecodes.c" + #line 1890 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2734 "Python/generated_cases.c.h" + #line 2733 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1896 "Python/bytecodes.c" + #line 1895 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2738 "Python/generated_cases.c.h" + #line 2737 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2743,15 +2742,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1900 "Python/bytecodes.c" + #line 1899 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2750 "Python/generated_cases.c.h" + #line 2749 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1903 "Python/bytecodes.c" + #line 1902 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2755 "Python/generated_cases.c.h" + #line 2754 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2760,29 +2759,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1907 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2768 "Python/generated_cases.c.h" + #line 2767 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1913 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" JUMPBY(oparg); - #line 2777 "Python/generated_cases.c.h" + #line 2776 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1917 "Python/bytecodes.c" + #line 1916 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2786 "Python/generated_cases.c.h" + #line 2785 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2790,7 +2789,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1923 "Python/bytecodes.c" + #line 1922 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2800,9 +2799,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2804 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1933 "Python/bytecodes.c" + #line 1932 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2810,14 +2809,14 @@ if (err < 0) goto pop_1_error; } } - #line 2814 "Python/generated_cases.c.h" + #line 2813 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1943 "Python/bytecodes.c" + #line 1942 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2827,9 +2826,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2831 "Python/generated_cases.c.h" + #line 2830 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1953 "Python/bytecodes.c" + #line 1952 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2837,67 +2836,67 @@ if (err < 0) goto pop_1_error; } } - #line 2841 "Python/generated_cases.c.h" + #line 2840 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1963 "Python/bytecodes.c" + #line 1962 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2850 "Python/generated_cases.c.h" + #line 2849 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1965 "Python/bytecodes.c" + #line 1964 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2858 "Python/generated_cases.c.h" + #line 2857 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1973 "Python/bytecodes.c" + #line 1972 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2871 "Python/generated_cases.c.h" + #line 2870 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1979 "Python/bytecodes.c" + #line 1978 "Python/bytecodes.c" } - #line 2875 "Python/generated_cases.c.h" + #line 2874 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1983 "Python/bytecodes.c" + #line 1982 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2888 "Python/generated_cases.c.h" + #line 2887 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1992 "Python/bytecodes.c" + #line 1991 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2901 "Python/generated_cases.c.h" + #line 2900 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2908,16 +2907,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2000 "Python/bytecodes.c" + #line 1999 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2917 "Python/generated_cases.c.h" + #line 2916 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2005 "Python/bytecodes.c" + #line 2004 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2925,7 +2924,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2929 "Python/generated_cases.c.h" + #line 2928 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2934,10 +2933,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2015 "Python/bytecodes.c" + #line 2014 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2941 "Python/generated_cases.c.h" + #line 2940 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2947,10 +2946,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2021 "Python/bytecodes.c" + #line 2020 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2954 "Python/generated_cases.c.h" + #line 2953 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2961,11 +2960,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2027 "Python/bytecodes.c" + #line 2026 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2969 "Python/generated_cases.c.h" + #line 2968 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2974,14 +2973,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2033 "Python/bytecodes.c" + #line 2032 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2981 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2036 "Python/bytecodes.c" + #line 2035 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2985 "Python/generated_cases.c.h" + #line 2984 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2989,7 +2988,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2040 "Python/bytecodes.c" + #line 2039 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3012,11 +3011,11 @@ if (iter == NULL) { goto error; } - #line 3016 "Python/generated_cases.c.h" + #line 3015 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2063 "Python/bytecodes.c" + #line 2062 "Python/bytecodes.c" } - #line 3020 "Python/generated_cases.c.h" + #line 3019 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3027,7 +3026,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2082 "Python/bytecodes.c" + #line 2081 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3058,7 +3057,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3062 "Python/generated_cases.c.h" + #line 3061 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3066,7 +3065,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2115 "Python/bytecodes.c" + #line 2114 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3092,14 +3091,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3096 "Python/generated_cases.c.h" + #line 3095 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2143 "Python/bytecodes.c" + #line 2142 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3119,7 +3118,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3123 "Python/generated_cases.c.h" + #line 3122 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3129,7 +3128,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2165 "Python/bytecodes.c" + #line 2164 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3149,7 +3148,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3153 "Python/generated_cases.c.h" + #line 3152 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3159,7 +3158,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2187 "Python/bytecodes.c" + #line 2186 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3177,7 +3176,7 @@ if (next == NULL) { goto error; } - #line 3181 "Python/generated_cases.c.h" + #line 3180 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3186,7 +3185,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2207 "Python/bytecodes.c" + #line 2206 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3201,14 +3200,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3205 "Python/generated_cases.c.h" + #line 3204 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2224 "Python/bytecodes.c" + #line 2223 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3231,16 +3230,16 @@ Py_DECREF(enter); goto error; } - #line 3235 "Python/generated_cases.c.h" + #line 3234 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2247 "Python/bytecodes.c" + #line 2246 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3244 "Python/generated_cases.c.h" + #line 3243 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3252,7 +3251,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2257 "Python/bytecodes.c" + #line 2256 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3278,16 +3277,16 @@ Py_DECREF(enter); goto error; } - #line 3282 "Python/generated_cases.c.h" + #line 3281 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2283 "Python/bytecodes.c" + #line 2282 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3291 "Python/generated_cases.c.h" + #line 3290 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3299,7 +3298,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2292 "Python/bytecodes.c" + #line 2291 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3320,7 +3319,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3324 "Python/generated_cases.c.h" + #line 3323 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3329,7 +3328,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2315 "Python/bytecodes.c" + #line 2314 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3339,7 +3338,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3343 "Python/generated_cases.c.h" + #line 3342 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3353,7 +3352,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2327 "Python/bytecodes.c" + #line 2326 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3370,7 +3369,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3374 "Python/generated_cases.c.h" + #line 3373 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3384,7 +3383,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2346 "Python/bytecodes.c" + #line 2345 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3394,7 +3393,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3398 "Python/generated_cases.c.h" + #line 3397 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3408,7 +3407,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2358 "Python/bytecodes.c" + #line 2357 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3422,7 +3421,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3426 "Python/generated_cases.c.h" + #line 3425 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3431,16 +3430,16 @@ } TARGET(KW_NAMES) { - #line 2374 "Python/bytecodes.c" + #line 2373 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3439 "Python/generated_cases.c.h" + #line 3438 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2380 "Python/bytecodes.c" + #line 2379 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3453,7 +3452,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3457 "Python/generated_cases.c.h" + #line 3456 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3463,7 +3462,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2425 "Python/bytecodes.c" + #line 2424 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3544,7 +3543,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3548 "Python/generated_cases.c.h" + #line 3547 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3556,7 +3555,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2512 "Python/bytecodes.c" + #line 2511 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3566,7 +3565,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3570 "Python/generated_cases.c.h" + #line 3569 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3575,7 +3574,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2524 "Python/bytecodes.c" + #line 2523 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3600,7 +3599,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3604 "Python/generated_cases.c.h" + #line 3603 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3609,7 +3608,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2551 "Python/bytecodes.c" + #line 2550 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3639,7 +3638,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3643 "Python/generated_cases.c.h" + #line 3642 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3647,7 +3646,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2583 "Python/bytecodes.c" + #line 2582 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3657,7 +3656,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3661 "Python/generated_cases.c.h" + #line 3660 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3670,7 +3669,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2595 "Python/bytecodes.c" + #line 2594 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3681,7 +3680,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3685 "Python/generated_cases.c.h" + #line 3684 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3695,7 +3694,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2609 "Python/bytecodes.c" + #line 2608 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3706,7 +3705,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3710 "Python/generated_cases.c.h" + #line 3709 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3720,7 +3719,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2623 "Python/bytecodes.c" + #line 2622 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3742,7 +3741,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3746 "Python/generated_cases.c.h" + #line 3745 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3756,7 +3755,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2648 "Python/bytecodes.c" + #line 2647 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3784,7 +3783,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3788 "Python/generated_cases.c.h" + #line 3787 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3798,7 +3797,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2679 "Python/bytecodes.c" + #line 2678 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3830,7 +3829,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3834 "Python/generated_cases.c.h" + #line 3833 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3844,7 +3843,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2714 "Python/bytecodes.c" + #line 2713 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3876,7 +3875,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3880 "Python/generated_cases.c.h" + #line 3879 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3890,7 +3889,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2749 "Python/bytecodes.c" + #line 2748 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3915,7 +3914,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3919 "Python/generated_cases.c.h" + #line 3918 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3928,7 +3927,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2776 "Python/bytecodes.c" + #line 2775 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3955,7 +3954,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3959 "Python/generated_cases.c.h" + #line 3958 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3967,7 +3966,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2806 "Python/bytecodes.c" + #line 2805 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3985,14 +3984,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3989 "Python/generated_cases.c.h" + #line 3988 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2826 "Python/bytecodes.c" + #line 2825 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4023,7 +4022,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4027 "Python/generated_cases.c.h" + #line 4026 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4036,7 +4035,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2860 "Python/bytecodes.c" + #line 2859 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4065,7 +4064,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4069 "Python/generated_cases.c.h" + #line 4068 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4078,7 +4077,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2892 "Python/bytecodes.c" + #line 2891 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4107,7 +4106,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4111 "Python/generated_cases.c.h" + #line 4110 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4120,7 +4119,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2924 "Python/bytecodes.c" + #line 2923 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4148,7 +4147,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4152 "Python/generated_cases.c.h" + #line 4151 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4158,9 +4157,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2955 "Python/bytecodes.c" + #line 2954 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4164 "Python/generated_cases.c.h" + #line 4163 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4169,7 +4168,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2959 "Python/bytecodes.c" + #line 2958 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4212,14 +4211,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4216 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3002 "Python/bytecodes.c" + #line 3001 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4223 "Python/generated_cases.c.h" + #line 4222 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4234,7 +4233,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3012 "Python/bytecodes.c" + #line 3011 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4263,14 +4262,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4267 "Python/generated_cases.c.h" + #line 4266 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3043 "Python/bytecodes.c" + #line 3042 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4291,7 +4290,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4295 "Python/generated_cases.c.h" + #line 4294 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4299,15 +4298,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3066 "Python/bytecodes.c" + #line 3065 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4305 "Python/generated_cases.c.h" + #line 4304 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3068 "Python/bytecodes.c" + #line 3067 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4311 "Python/generated_cases.c.h" + #line 4310 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4318,7 +4317,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3072 "Python/bytecodes.c" + #line 3071 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4353,7 +4352,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4357 "Python/generated_cases.c.h" + #line 4356 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4362,10 +4361,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3109 "Python/bytecodes.c" + #line 3108 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4369 "Python/generated_cases.c.h" + #line 4368 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4377,7 +4376,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3114 "Python/bytecodes.c" + #line 3113 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4392,12 +4391,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4396 "Python/generated_cases.c.h" + #line 4395 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3129 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4401 "Python/generated_cases.c.h" + #line 4400 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4407,22 +4406,21 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3134 "Python/bytecodes.c" + #line 3133 "Python/bytecodes.c" assert(oparg >= 2); - #line 4413 "Python/generated_cases.c.h" + #line 4412 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3138 "Python/bytecodes.c" + #line 3137 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; @@ -4437,11 +4435,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4441 "Python/generated_cases.c.h" + #line 4439 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3161 "Python/bytecodes.c" + #line 3159 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4453,26 +4451,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4457 "Python/generated_cases.c.h" + #line 4455 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3176 "Python/bytecodes.c" + #line 3174 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4463 "Python/generated_cases.c.h" + #line 4461 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3181 "Python/bytecodes.c" + #line 3179 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4470 "Python/generated_cases.c.h" + #line 4468 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3186 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4481,12 +4479,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4485 "Python/generated_cases.c.h" + #line 4483 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3197 "Python/bytecodes.c" + #line 3195 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4495,12 +4493,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4499 "Python/generated_cases.c.h" + #line 4497 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3208 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4513,12 +4511,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4517 "Python/generated_cases.c.h" + #line 4515 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3223 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4531,22 +4529,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4535 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3238 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4546 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3246 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4552 "Python/generated_cases.c.h" + #line 4550 "Python/generated_cases.c.h" } From 6611c72929d2acb1915fa1ecc32b5f81c2d347bd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 15:30:08 +0000 Subject: [PATCH 072/116] Fixups from code review --- Python/instrumentation.c | 6 +++--- Python/legacy_tracing.c | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0147b4c35f1647..4a1fde044f10bf 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -612,8 +612,8 @@ de_instrument(PyCodeObject *code, int offset, int event) case INSTRUMENTED_LINE: { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; - int orignal_opcode = lines->original_opcode; - int deinstrumented = DE_INSTRUMENT[orignal_opcode]; + int original_opcode = lines->original_opcode; + int deinstrumented = DE_INSTRUMENT[original_opcode]; if (deinstrumented) { lines->original_opcode = opcode = deinstrumented; } @@ -694,7 +694,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) } } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for snaity check */ + /* Keep things clean for sanity check */ code->_co_monitoring->per_instruction_opcodes[offset] = 0; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 514afc00f7adf0..a0a47fa6e6abea 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,13 +4,8 @@ #include #include "Python.h" -#include "opcode.h" #include "pycore_ceval.h" -#include "pycore_instruments.h" #include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pymem.h" -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" typedef struct _PyLegacyEventHandler { @@ -40,7 +35,6 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); From 50d28f16f492dbc6e6f70daad3cc1ac1c709b904 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 21:26:27 +0000 Subject: [PATCH 073/116] Improve debugging output and make a few things more robust. --- Include/internal/pycore_opcode.h | 34 +- Include/opcode.h | 111 +++--- Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 2 + Objects/codeobject.c | 27 +- Programs/test_frozenmain.h | 16 +- Python/bytecodes.c | 5 + Python/generated_cases.c.h | 550 ++++++++++++++------------- Python/instrumentation.c | 131 ++++--- Python/opcode_metadata.h | 5 + Python/opcode_targets.h | 32 +- 11 files changed, 484 insertions(+), 431 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 53a446e378d143..b8d26b663353b0 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -206,6 +206,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, [RERAISE] = RERAISE, + [RESERVED] = RESERVED, [RESUME] = RESUME, [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, @@ -262,6 +263,7 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [UNARY_INVERT] = "UNARY_INVERT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [RESERVED] = "RESERVED", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -269,20 +271,20 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", @@ -293,7 +295,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -301,39 +302,39 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -356,9 +357,9 @@ static const char *const _PyOpcode_OpName[263] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -386,9 +387,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -398,14 +399,14 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [160] = "<160>", [161] = "<161>", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", @@ -512,7 +513,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 160: \ case 161: \ case 166: \ case 167: \ diff --git a/Include/opcode.h b/Include/opcode.h index e619ed968a8e0a..701d7f81837e5e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -18,6 +18,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define RESERVED 17 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -148,61 +149,61 @@ extern "C" { #define BINARY_OP_MULTIPLY_FLOAT 13 #define BINARY_OP_MULTIPLY_INT 14 #define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 17 -#define BINARY_SUBSCR_DICT 18 -#define BINARY_SUBSCR_GETITEM 19 -#define BINARY_SUBSCR_LIST_INT 20 -#define BINARY_SUBSCR_TUPLE_INT 21 -#define CALL_PY_EXACT_ARGS 22 -#define CALL_PY_WITH_DEFAULTS 23 -#define CALL_BOUND_METHOD_EXACT_ARGS 24 -#define CALL_BUILTIN_CLASS 28 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 -#define CALL_NO_KW_BUILTIN_FAST 38 -#define CALL_NO_KW_BUILTIN_O 39 -#define CALL_NO_KW_ISINSTANCE 40 -#define CALL_NO_KW_LEN 41 -#define CALL_NO_KW_LIST_APPEND 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 -#define CALL_NO_KW_STR_1 46 -#define CALL_NO_KW_TUPLE_1 47 -#define CALL_NO_KW_TYPE_1 48 -#define COMPARE_OP_FLOAT 56 -#define COMPARE_OP_INT 57 -#define COMPARE_OP_STR 58 -#define FOR_ITER_LIST 59 -#define FOR_ITER_TUPLE 62 -#define FOR_ITER_RANGE 63 -#define FOR_ITER_GEN 64 -#define LOAD_ATTR_CLASS 65 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_PROPERTY 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_WITH_HINT 76 -#define LOAD_ATTR_METHOD_LAZY_DICT 77 -#define LOAD_ATTR_METHOD_NO_DICT 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_CONST__LOAD_FAST 80 -#define LOAD_FAST__LOAD_CONST 81 -#define LOAD_FAST__LOAD_FAST 82 -#define LOAD_GLOBAL_BUILTIN 84 -#define LOAD_GLOBAL_MODULE 86 -#define STORE_ATTR_INSTANCE_VALUE 87 -#define STORE_ATTR_SLOT 88 -#define STORE_ATTR_WITH_HINT 111 -#define STORE_FAST__LOAD_FAST 112 -#define STORE_FAST__STORE_FAST 113 -#define STORE_SUBSCR_DICT 141 -#define STORE_SUBSCR_LIST_INT 143 -#define UNPACK_SEQUENCE_LIST 153 -#define UNPACK_SEQUENCE_TUPLE 154 -#define UNPACK_SEQUENCE_TWO_TUPLE 158 -#define SEND_GEN 159 +#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define CALL_BOUND_METHOD_EXACT_ARGS 28 +#define CALL_BUILTIN_CLASS 29 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 +#define CALL_NO_KW_BUILTIN_FAST 39 +#define CALL_NO_KW_BUILTIN_O 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_LIST_APPEND 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 56 +#define COMPARE_OP_FLOAT 57 +#define COMPARE_OP_INT 58 +#define COMPARE_OP_STR 59 +#define FOR_ITER_LIST 62 +#define FOR_ITER_TUPLE 63 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 70 +#define LOAD_ATTR_MODULE 72 +#define LOAD_ATTR_PROPERTY 73 +#define LOAD_ATTR_SLOT 76 +#define LOAD_ATTR_WITH_HINT 77 +#define LOAD_ATTR_METHOD_LAZY_DICT 78 +#define LOAD_ATTR_METHOD_NO_DICT 79 +#define LOAD_ATTR_METHOD_WITH_VALUES 80 +#define LOAD_CONST__LOAD_FAST 81 +#define LOAD_FAST__LOAD_CONST 82 +#define LOAD_FAST__LOAD_FAST 84 +#define LOAD_GLOBAL_BUILTIN 86 +#define LOAD_GLOBAL_MODULE 87 +#define STORE_ATTR_INSTANCE_VALUE 88 +#define STORE_ATTR_SLOT 111 +#define STORE_ATTR_WITH_HINT 112 +#define STORE_FAST__LOAD_FAST 113 +#define STORE_FAST__STORE_FAST 141 +#define STORE_SUBSCR_DICT 143 +#define STORE_SUBSCR_LIST_INT 153 +#define UNPACK_SEQUENCE_LIST 154 +#define UNPACK_SEQUENCE_TUPLE 158 +#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define SEND_GEN 160 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f16e06c018ccf0..aa00a10474cdc5 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,7 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 2057d451f0eb26..3ca4173e6bd83b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -92,6 +92,8 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_INVERT', 15) +def_op('RESERVED', 17) + def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) def_op('STORE_SLICE', 27) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 171f514b04575e..024bd7bf78daa5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1487,9 +1487,12 @@ deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; - while (caches--) { - instructions[++i].op.code = CACHE; - instructions[i].op.arg = 0; + if (caches) { + instructions[i+1].cache = adaptive_counter_warmup(); + for (int j = 2; j <= caches; j++) { + instructions[i+j].cache = 0; + } + i += caches; } } } @@ -2288,6 +2291,24 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } + _PyCoMonitoringData *data = co->_co_monitoring; + if (data) { + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + } } int diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8e5055bd7bceb1..57635a398d1df0 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,16 +3,16 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,182,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, - 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, + 100,2,171,1,17,0,0,0,0,0,0,0,1,0,2,0, + 101,2,100,3,101,0,106,6,17,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,171,2,17,0,0,0, + 0,0,0,0,1,0,2,0,101,1,106,8,17,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 0,0,0,0,0,0,0,0,100,4,25,0,0,0,0,0, - 0,0,0,0,90,5,100,5,68,0,93,23,0,0,90,6, + 17,0,0,0,0,0,0,0,100,4,25,0,17,0,0,0, + 0,0,0,0,90,5,100,5,68,0,93,23,17,0,90,6, 2,0,101,2,100,6,101,6,155,0,100,7,101,5,101,6, - 25,0,0,0,0,0,0,0,0,0,155,0,157,4,171,1, - 0,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, + 25,0,17,0,0,0,0,0,0,0,155,0,157,4,171,1, + 17,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f8df0ccb030889..6768513f8d4799 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1234,6 +1234,7 @@ dummy_func( DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -3244,6 +3245,10 @@ dummy_func( Py_UNREACHABLE(); } + inst(RESERVED, (--)) { + assert(0 && "Executing RESERVED instruction."); + } + // END BYTECODES // diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 08a294e4803e9e..1c9f94ef40a2e1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1760,6 +1760,7 @@ DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -1769,7 +1770,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1773 "Python/generated_cases.c.h" + #line 1774 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1779,16 +1780,16 @@ } TARGET(DELETE_FAST) { - #line 1249 "Python/bytecodes.c" + #line 1250 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1787 "Python/generated_cases.c.h" + #line 1788 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1255 "Python/bytecodes.c" + #line 1256 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1797,12 +1798,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1801 "Python/generated_cases.c.h" + #line 1802 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1266 "Python/bytecodes.c" + #line 1267 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1813,13 +1814,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1817 "Python/generated_cases.c.h" + #line 1818 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1279 "Python/bytecodes.c" + #line 1280 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1851,7 +1852,7 @@ } Py_INCREF(value); } - #line 1855 "Python/generated_cases.c.h" + #line 1856 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1859,7 +1860,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1313 "Python/bytecodes.c" + #line 1314 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1867,7 +1868,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1871 "Python/generated_cases.c.h" + #line 1872 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1875,18 +1876,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1323 "Python/bytecodes.c" + #line 1324 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1884 "Python/generated_cases.c.h" + #line 1885 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1330 "Python/bytecodes.c" + #line 1331 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1897,22 +1898,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1901 "Python/generated_cases.c.h" + #line 1902 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1343 "Python/bytecodes.c" + #line 1344 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1910 "Python/generated_cases.c.h" + #line 1911 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1345 "Python/bytecodes.c" + #line 1346 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1916 "Python/generated_cases.c.h" + #line 1917 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1922,10 +1923,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1349 "Python/bytecodes.c" + #line 1350 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1929 "Python/generated_cases.c.h" + #line 1930 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1935,10 +1936,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1354 "Python/bytecodes.c" + #line 1355 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1942 "Python/generated_cases.c.h" + #line 1943 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1948,7 +1949,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1359 "Python/bytecodes.c" + #line 1360 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1959,13 +1960,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1963 "Python/generated_cases.c.h" + #line 1964 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1370 "Python/bytecodes.c" + #line 1371 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1969 "Python/generated_cases.c.h" + #line 1970 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1974,13 +1975,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1377 "Python/bytecodes.c" + #line 1378 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1980 "Python/generated_cases.c.h" + #line 1981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1379 "Python/bytecodes.c" + #line 1380 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1984 "Python/generated_cases.c.h" + #line 1985 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1988,7 +1989,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1383 "Python/bytecodes.c" + #line 1384 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2003,7 +2004,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2007 "Python/generated_cases.c.h" + #line 2008 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2013,7 +2014,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1400 "Python/bytecodes.c" + #line 1401 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2021,13 +2022,13 @@ if (map == NULL) goto error; - #line 2025 "Python/generated_cases.c.h" + #line 2026 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1408 "Python/bytecodes.c" + #line 1409 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2031 "Python/generated_cases.c.h" + #line 2032 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2035,7 +2036,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1412 "Python/bytecodes.c" + #line 1413 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2075,7 +2076,7 @@ Py_DECREF(ann_dict); } } - #line 2079 "Python/generated_cases.c.h" + #line 2080 "Python/generated_cases.c.h" DISPATCH(); } @@ -2083,7 +2084,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1454 "Python/bytecodes.c" + #line 1455 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2093,14 +2094,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2097 "Python/generated_cases.c.h" + #line 2098 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1464 "Python/bytecodes.c" + #line 1465 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2104 "Python/generated_cases.c.h" + #line 2105 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2108,7 +2109,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1468 "Python/bytecodes.c" + #line 1469 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2116,12 +2117,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2120 "Python/generated_cases.c.h" + #line 2121 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1476 "Python/bytecodes.c" + #line 1477 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2125 "Python/generated_cases.c.h" + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2129,17 +2130,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1482 "Python/bytecodes.c" + #line 1483 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2138 "Python/generated_cases.c.h" + #line 2139 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1487 "Python/bytecodes.c" + #line 1488 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2143 "Python/generated_cases.c.h" + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2149,13 +2150,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1494 "Python/bytecodes.c" + #line 1495 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2159 "Python/generated_cases.c.h" + #line 2160 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2167,7 +2168,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1517 "Python/bytecodes.c" + #line 1518 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2201,9 +2202,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2205 "Python/generated_cases.c.h" + #line 2206 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1551 "Python/bytecodes.c" + #line 1552 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2212,12 +2213,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2216 "Python/generated_cases.c.h" + #line 2217 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1560 "Python/bytecodes.c" + #line 1561 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2221 "Python/generated_cases.c.h" + #line 2222 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2231,7 +2232,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1565 "Python/bytecodes.c" + #line 1566 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2244,7 +2245,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2248 "Python/generated_cases.c.h" + #line 2249 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2259,7 +2260,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1581 "Python/bytecodes.c" + #line 1582 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2272,7 +2273,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2276 "Python/generated_cases.c.h" + #line 2277 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2287,7 +2288,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1597 "Python/bytecodes.c" + #line 1598 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2314,7 +2315,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2318 "Python/generated_cases.c.h" + #line 2319 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2329,7 +2330,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1627 "Python/bytecodes.c" + #line 1628 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2339,7 +2340,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2343 "Python/generated_cases.c.h" + #line 2344 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2354,7 +2355,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1640 "Python/bytecodes.c" + #line 1641 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2366,7 +2367,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2370 "Python/generated_cases.c.h" + #line 2371 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2380,7 +2381,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1655 "Python/bytecodes.c" + #line 1656 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2403,7 +2404,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2407 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2411,7 +2412,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1680 "Python/bytecodes.c" + #line 1681 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2436,7 +2437,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2440 "Python/generated_cases.c.h" + #line 2441 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2444,7 +2445,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1707 "Python/bytecodes.c" + #line 1708 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2462,7 +2463,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2466 "Python/generated_cases.c.h" + #line 2467 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2473,7 +2474,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1727 "Python/bytecodes.c" + #line 1728 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2512,7 +2513,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2516 "Python/generated_cases.c.h" + #line 2517 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2523,7 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1768 "Python/bytecodes.c" + #line 1769 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2533,7 +2534,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2537 "Python/generated_cases.c.h" + #line 2538 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2545,7 +2546,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1787 "Python/bytecodes.c" + #line 1788 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2558,12 +2559,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2562 "Python/generated_cases.c.h" + #line 2563 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1800 "Python/bytecodes.c" + #line 1801 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2567 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2574,7 +2575,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1804 "Python/bytecodes.c" + #line 1805 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2586,7 +2587,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2590 "Python/generated_cases.c.h" + #line 2591 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2597,7 +2598,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1819 "Python/bytecodes.c" + #line 1820 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2613,7 +2614,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2617 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2624,7 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1838 "Python/bytecodes.c" + #line 1839 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2637,7 +2638,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2641 "Python/generated_cases.c.h" + #line 2642 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2648,14 +2649,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1853 "Python/bytecodes.c" + #line 1854 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2654 "Python/generated_cases.c.h" + #line 2655 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1855 "Python/bytecodes.c" + #line 1856 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2659 "Python/generated_cases.c.h" + #line 2660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2665,15 +2666,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1859 "Python/bytecodes.c" + #line 1860 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2671 "Python/generated_cases.c.h" + #line 2672 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1861 "Python/bytecodes.c" + #line 1862 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2677 "Python/generated_cases.c.h" + #line 2678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2684,12 +2685,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1866 "Python/bytecodes.c" + #line 1867 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2690 "Python/generated_cases.c.h" + #line 2691 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1868 "Python/bytecodes.c" + #line 1869 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2697,10 +2698,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2701 "Python/generated_cases.c.h" + #line 2702 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1876 "Python/bytecodes.c" + #line 1877 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2709,7 +2710,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2713 "Python/generated_cases.c.h" + #line 2714 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2719,21 +2720,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1887 "Python/bytecodes.c" + #line 1888 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2726 "Python/generated_cases.c.h" + #line 2727 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1890 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2733 "Python/generated_cases.c.h" + #line 2734 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1895 "Python/bytecodes.c" + #line 1896 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2737 "Python/generated_cases.c.h" + #line 2738 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2742,15 +2743,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1899 "Python/bytecodes.c" + #line 1900 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2749 "Python/generated_cases.c.h" + #line 2750 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1902 "Python/bytecodes.c" + #line 1903 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2754 "Python/generated_cases.c.h" + #line 2755 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2759,29 +2760,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1906 "Python/bytecodes.c" + #line 1907 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2767 "Python/generated_cases.c.h" + #line 2768 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1912 "Python/bytecodes.c" + #line 1913 "Python/bytecodes.c" JUMPBY(oparg); - #line 2776 "Python/generated_cases.c.h" + #line 2777 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1916 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2785 "Python/generated_cases.c.h" + #line 2786 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2789,7 +2790,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1922 "Python/bytecodes.c" + #line 1923 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2799,9 +2800,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2803 "Python/generated_cases.c.h" + #line 2804 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1932 "Python/bytecodes.c" + #line 1933 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2809,14 +2810,14 @@ if (err < 0) goto pop_1_error; } } - #line 2813 "Python/generated_cases.c.h" + #line 2814 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1942 "Python/bytecodes.c" + #line 1943 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2826,9 +2827,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2830 "Python/generated_cases.c.h" + #line 2831 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1952 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2836,67 +2837,67 @@ if (err < 0) goto pop_1_error; } } - #line 2840 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1962 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2849 "Python/generated_cases.c.h" + #line 2850 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1964 "Python/bytecodes.c" + #line 1965 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2857 "Python/generated_cases.c.h" + #line 2858 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1972 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2870 "Python/generated_cases.c.h" + #line 2871 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1978 "Python/bytecodes.c" + #line 1979 "Python/bytecodes.c" } - #line 2874 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1982 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2887 "Python/generated_cases.c.h" + #line 2888 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1991 "Python/bytecodes.c" + #line 1992 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2900 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2907,16 +2908,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1999 "Python/bytecodes.c" + #line 2000 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2916 "Python/generated_cases.c.h" + #line 2917 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2004 "Python/bytecodes.c" + #line 2005 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2924,7 +2925,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2928 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2933,10 +2934,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2014 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2940 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2946,10 +2947,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2020 "Python/bytecodes.c" + #line 2021 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2953 "Python/generated_cases.c.h" + #line 2954 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2960,11 +2961,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2026 "Python/bytecodes.c" + #line 2027 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2968 "Python/generated_cases.c.h" + #line 2969 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2973,14 +2974,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2032 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2980 "Python/generated_cases.c.h" + #line 2981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2035 "Python/bytecodes.c" + #line 2036 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2984 "Python/generated_cases.c.h" + #line 2985 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2988,7 +2989,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2039 "Python/bytecodes.c" + #line 2040 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3011,11 +3012,11 @@ if (iter == NULL) { goto error; } - #line 3015 "Python/generated_cases.c.h" + #line 3016 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2062 "Python/bytecodes.c" + #line 2063 "Python/bytecodes.c" } - #line 3019 "Python/generated_cases.c.h" + #line 3020 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3026,7 +3027,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2081 "Python/bytecodes.c" + #line 2082 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3057,7 +3058,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3061 "Python/generated_cases.c.h" + #line 3062 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3065,7 +3066,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2114 "Python/bytecodes.c" + #line 2115 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3091,14 +3092,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3095 "Python/generated_cases.c.h" + #line 3096 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2142 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3118,7 +3119,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3122 "Python/generated_cases.c.h" + #line 3123 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3128,7 +3129,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2164 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3148,7 +3149,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3152 "Python/generated_cases.c.h" + #line 3153 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3158,7 +3159,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2186 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3176,7 +3177,7 @@ if (next == NULL) { goto error; } - #line 3180 "Python/generated_cases.c.h" + #line 3181 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3185,7 +3186,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2206 "Python/bytecodes.c" + #line 2207 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3200,14 +3201,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3204 "Python/generated_cases.c.h" + #line 3205 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2223 "Python/bytecodes.c" + #line 2224 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3230,16 +3231,16 @@ Py_DECREF(enter); goto error; } - #line 3234 "Python/generated_cases.c.h" + #line 3235 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2246 "Python/bytecodes.c" + #line 2247 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3243 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3251,7 +3252,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2256 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3277,16 +3278,16 @@ Py_DECREF(enter); goto error; } - #line 3281 "Python/generated_cases.c.h" + #line 3282 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2282 "Python/bytecodes.c" + #line 2283 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3290 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3298,7 +3299,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2291 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3319,7 +3320,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3323 "Python/generated_cases.c.h" + #line 3324 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3328,7 +3329,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2314 "Python/bytecodes.c" + #line 2315 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3338,7 +3339,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3342 "Python/generated_cases.c.h" + #line 3343 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3352,7 +3353,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2326 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3369,7 +3370,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3373 "Python/generated_cases.c.h" + #line 3374 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3383,7 +3384,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2345 "Python/bytecodes.c" + #line 2346 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3393,7 +3394,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3397 "Python/generated_cases.c.h" + #line 3398 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3407,7 +3408,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2357 "Python/bytecodes.c" + #line 2358 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3421,7 +3422,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3426 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3430,16 +3431,16 @@ } TARGET(KW_NAMES) { - #line 2373 "Python/bytecodes.c" + #line 2374 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3438 "Python/generated_cases.c.h" + #line 3439 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2379 "Python/bytecodes.c" + #line 2380 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3452,7 +3453,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3456 "Python/generated_cases.c.h" + #line 3457 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3462,7 +3463,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2424 "Python/bytecodes.c" + #line 2425 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3543,7 +3544,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3547 "Python/generated_cases.c.h" + #line 3548 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3555,7 +3556,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2511 "Python/bytecodes.c" + #line 2512 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3565,7 +3566,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3569 "Python/generated_cases.c.h" + #line 3570 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3574,7 +3575,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2523 "Python/bytecodes.c" + #line 2524 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3599,7 +3600,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3603 "Python/generated_cases.c.h" + #line 3604 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3608,7 +3609,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2550 "Python/bytecodes.c" + #line 2551 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3638,7 +3639,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3642 "Python/generated_cases.c.h" + #line 3643 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3646,7 +3647,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2582 "Python/bytecodes.c" + #line 2583 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3656,7 +3657,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3660 "Python/generated_cases.c.h" + #line 3661 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3669,7 +3670,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2594 "Python/bytecodes.c" + #line 2595 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3680,7 +3681,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3684 "Python/generated_cases.c.h" + #line 3685 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3694,7 +3695,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2608 "Python/bytecodes.c" + #line 2609 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3705,7 +3706,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3709 "Python/generated_cases.c.h" + #line 3710 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3719,7 +3720,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2622 "Python/bytecodes.c" + #line 2623 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3741,7 +3742,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3745 "Python/generated_cases.c.h" + #line 3746 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3755,7 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2647 "Python/bytecodes.c" + #line 2648 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3783,7 +3784,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3787 "Python/generated_cases.c.h" + #line 3788 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3797,7 +3798,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2678 "Python/bytecodes.c" + #line 2679 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3829,7 +3830,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3833 "Python/generated_cases.c.h" + #line 3834 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3843,7 +3844,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2713 "Python/bytecodes.c" + #line 2714 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3875,7 +3876,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3879 "Python/generated_cases.c.h" + #line 3880 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3889,7 +3890,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2748 "Python/bytecodes.c" + #line 2749 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3914,7 +3915,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3918 "Python/generated_cases.c.h" + #line 3919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3927,7 +3928,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2775 "Python/bytecodes.c" + #line 2776 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3954,7 +3955,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3958 "Python/generated_cases.c.h" + #line 3959 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3966,7 +3967,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2805 "Python/bytecodes.c" + #line 2806 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3984,14 +3985,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3988 "Python/generated_cases.c.h" + #line 3989 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2825 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4022,7 +4023,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4026 "Python/generated_cases.c.h" + #line 4027 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4035,7 +4036,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2859 "Python/bytecodes.c" + #line 2860 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4064,7 +4065,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4068 "Python/generated_cases.c.h" + #line 4069 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4077,7 +4078,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2891 "Python/bytecodes.c" + #line 2892 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4106,7 +4107,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4110 "Python/generated_cases.c.h" + #line 4111 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4119,7 +4120,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2923 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4147,7 +4148,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4151 "Python/generated_cases.c.h" + #line 4152 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4157,9 +4158,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2954 "Python/bytecodes.c" + #line 2955 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4163 "Python/generated_cases.c.h" + #line 4164 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4168,7 +4169,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2958 "Python/bytecodes.c" + #line 2959 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4211,14 +4212,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4215 "Python/generated_cases.c.h" + #line 4216 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3001 "Python/bytecodes.c" + #line 3002 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4222 "Python/generated_cases.c.h" + #line 4223 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4233,7 +4234,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3011 "Python/bytecodes.c" + #line 3012 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4262,14 +4263,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4266 "Python/generated_cases.c.h" + #line 4267 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3042 "Python/bytecodes.c" + #line 3043 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4290,7 +4291,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4294 "Python/generated_cases.c.h" + #line 4295 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4298,15 +4299,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3065 "Python/bytecodes.c" + #line 3066 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4304 "Python/generated_cases.c.h" + #line 4305 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3067 "Python/bytecodes.c" + #line 3068 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4310 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4317,7 +4318,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3071 "Python/bytecodes.c" + #line 3072 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4352,7 +4353,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4356 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4361,10 +4362,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3108 "Python/bytecodes.c" + #line 3109 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4376,7 +4377,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3113 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4391,12 +4392,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4395 "Python/generated_cases.c.h" + #line 4396 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3128 "Python/bytecodes.c" + #line 3129 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4400 "Python/generated_cases.c.h" + #line 4401 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4406,16 +4407,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3133 "Python/bytecodes.c" + #line 3134 "Python/bytecodes.c" assert(oparg >= 2); - #line 4412 "Python/generated_cases.c.h" + #line 4413 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3137 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4435,11 +4436,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4439 "Python/generated_cases.c.h" + #line 4440 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3159 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4451,26 +4452,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4455 "Python/generated_cases.c.h" + #line 4456 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3174 "Python/bytecodes.c" + #line 3175 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4461 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3179 "Python/bytecodes.c" + #line 3180 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4468 "Python/generated_cases.c.h" + #line 4469 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3184 "Python/bytecodes.c" + #line 3185 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4479,12 +4480,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4483 "Python/generated_cases.c.h" + #line 4484 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3195 "Python/bytecodes.c" + #line 3196 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4493,12 +4494,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4497 "Python/generated_cases.c.h" + #line 4498 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3206 "Python/bytecodes.c" + #line 3207 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4511,12 +4512,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4515 "Python/generated_cases.c.h" + #line 4516 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3221 "Python/bytecodes.c" + #line 3222 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4529,22 +4530,29 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4534 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3236 "Python/bytecodes.c" + #line 3237 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4544 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3244 "Python/bytecodes.c" + #line 3245 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4550 "Python/generated_cases.c.h" + #line 4551 "Python/generated_cases.c.h" + } + + TARGET(RESERVED) { + #line 3249 "Python/bytecodes.c" + assert(0 && "Executing RESERVED instruction."); + #line 4557 "Python/generated_cases.c.h" + DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4a1fde044f10bf..59f500c14570e4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -// #define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -144,6 +144,7 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { static inline bool is_instrumented(int opcode) { assert(opcode != 0); + assert(opcode != RESERVED); return INSTRUMENTED_OPCODES[opcode] == opcode; } @@ -271,25 +272,48 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) * so add a couple of utilities */ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } + int opcode = _PyCode_CODE(code)[offset].op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + assert(opcode != INSTRUMENTED_INSTRUCTION); + assert(opcode != INSTRUMENTED_LINE); int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { return deinstrumented; } assert(_PyOpcode_Deopt[opcode] != 0); + assert(opcode != RESERVED); return _PyOpcode_Deopt[opcode]; } +int instruction_length(PyCodeObject *code, int offset) +{ + int opcode = _PyCode_CODE(code)[offset].op.code; + assert(opcode != 0); + assert(opcode != RESERVED); + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + opcode = deinstrumented; + } + else { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(opcode != 0); + assert(!is_instrumented(opcode)); + assert(opcode == _PyOpcode_Deopt[opcode]); + return 1 + _PyOpcode_Caches[opcode]; +} + typedef struct _Instruction { uint8_t extended_args; uint8_t deinstrumented_opcode; @@ -329,9 +353,7 @@ read_instruction(PyCodeObject *code, int offset) else { result.deinstrumented_opcode = opcode; } - int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; - /* TO DO -- Use instruction length, not cache count table */ - result.length += _PyOpcode_Caches[base]; + result.length = result.extended_args + instruction_length(code, offset); return result; } @@ -440,6 +462,8 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { _PyCoMonitoringData *data = code->_co_monitoring; fprintf(out, "\n"); + PyObject_Print(code->co_name, out, Py_PRINT_RAW); + fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; @@ -449,7 +473,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; - for (int i = 0; i < code_len; i++) { + for (int i = 0; i < code_len; i += instruction_length(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; if (i == star) { @@ -463,9 +487,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_per_instruction(code, data, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); - /* TO DO -- Use instruction length, not cache count */ - int base_opcode = get_base_opcode_best_attempt(code, i); - i += _PyOpcode_Caches[base_opcode]; + ; } if (!starred && star >= 0) { fprintf(out, "Error offset not at valid instruction offset: %d\n", star); @@ -545,11 +567,6 @@ sanity_check_instrumentation(PyCodeObject *code) event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } CHECK(active_monitors.tools[event] != 0); - CHECK(inst.deinstrumented_opcode != END_FOR); - } - if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { - CHECK(_PyCode_CODE(code)[i+2].op.code == POP_JUMP_IF_FALSE || - _PyCode_CODE(code)[i+2].op.code == POP_JUMP_IF_TRUE); } if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); @@ -647,14 +664,13 @@ de_instrument_line(PyCodeObject *code, int i) } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; - assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->op.code = original_opcode; } else { - original_opcode = _PyOpcode_Deopt[original_opcode]; - assert(original_opcode != 0); + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -672,51 +688,53 @@ de_instrument_line(PyCodeObject *code, int i) static void -de_instrument_per_instruction(PyCodeObject *code, int offset) +de_instrument_per_instruction(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; - int opcode = _Py_OPCODE(*instr); + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + CHECK(original_opcode != 0); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->op.code = original_opcode; } else { - int base_opcode = _PyOpcode_Deopt[original_opcode]; - assert(base_opcode != 0); - instr->op.code = base_opcode; - if (_PyOpcode_Caches[base_opcode]) { + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[offset] = 0; + code->_co_monitoring->per_instruction_opcodes[i] = 0; } static void -instrument(PyCodeObject *code, int offset) +instrument(PyCodeObject *code, int i) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + CHECK(opcode != INSTRUMENTED_INSTRUCTION); if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; opcode = lines->original_opcode; - assert(!is_instrumented(opcode)); + CHECK(opcode != 0); + CHECK(!is_instrumented(opcode)); + CHECK(opcode == _PyOpcode_Deopt[opcode]); lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; + CHECK(lines->original_opcode != 0); } else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; assert(instrumented); instr->op.code = instrumented; - /* TO DO -- Use instruction length, not cache count */ if (_PyOpcode_Caches[opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -729,6 +747,7 @@ instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + CHECK(opcode != INSTRUMENTED_INSTRUCTION); if (opcode == INSTRUMENTED_LINE) { return; } @@ -739,10 +758,9 @@ instrument_line(PyCodeObject *code, int i) } else { assert(opcode != 0); - assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); lines->original_opcode = _PyOpcode_Deopt[opcode]; - + CHECK(lines->original_opcode != 0); } assert(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; @@ -1343,10 +1361,11 @@ initialize_lines(PyCodeObject *code) } int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; ) { - Instruction inst = read_instruction(code, i); + int opcode = _Py_GetBaseOpcode(code, i); int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); line_data[i].line_delta = compute_line_delta(code, i, line); - switch (inst.deinstrumented_opcode) { + int length = instruction_length(code, i); + switch (opcode) { case END_ASYNC_FOR: case END_FOR: case END_SEND: @@ -1367,11 +1386,11 @@ initialize_lines(PyCodeObject *code) current_line = line; } } - for (int j = 1; j < inst.length; j++) { + for (int j = 1; j < length; j++) { line_data[i+j].original_opcode = 0; line_data[i+j].line_delta = NO_LINE; } - switch (inst.deinstrumented_opcode) { + switch (opcode) { case RETURN_VALUE: case RAISE_VARARGS: case RERAISE: @@ -1379,7 +1398,7 @@ initialize_lines(PyCodeObject *code) * should be treated as different lines */ current_line = -1; } - i += inst.length; + i += length; } } @@ -1533,12 +1552,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } /* Insert instrumentation */ - for (int i = 0; i < code_len; i++) { + for (int i = 0; i < code_len; i+= instruction_length(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - if (is_super_instruction(opcode)) { - instr->op.code = _PyOpcode_Deopt[opcode]; + if (is_super_instruction(instr->op.code)) { + instr->op.code = _PyOpcode_Deopt[instr->op.code]; } + CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; @@ -1558,19 +1577,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_tools(code, i, event, new_tools); } } - while (base_opcode == EXTENDED_ARG) { - i++; - base_opcode = _Py_GetBaseOpcode(code, i); - } - i += _PyOpcode_Caches[base_opcode]; - } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1579,17 +1591,16 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_line_tools(code, i, new_line_tools); } } - i += inst.length; + i += instruction_length(code, i); } } uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (new_per_instruction_tools | removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { - Instruction inst = read_instruction(code, i); - int opcode = _PyOpcode_Deopt[inst.deinstrumented_opcode]; + int opcode = _Py_GetBaseOpcode(code, i); if (opcode == RESUME || opcode == END_FOR) { - i += inst.length; + i += instruction_length(code, i); continue; } if (removed_per_instruction_tools) { @@ -1598,7 +1609,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } - i += inst.length; + i += instruction_length(code, i); } } #ifdef INSTRUMENT_DEBUG diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 8788d44f110f35..7134bbf8f7ac0f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -379,6 +379,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -761,6 +763,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -962,5 +966,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, [EXTENDED_ARG] = { true, INSTR_FMT_IB }, [CACHE] = { true, INSTR_FMT_IX }, + [RESERVED] = { true, INSTR_FMT_IX }, }; #endif diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 81882e08ebb9d1..9d6616666f7ac1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -16,6 +16,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_UNARY_INVERT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_RESERVED, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, - &&TARGET_FOR_ITER_LIST, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,15 +152,15 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, From 7165f52c4c3e759a0542190848d3f2ab2d5b5b21 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 00:43:02 +0000 Subject: [PATCH 074/116] Fix up frame.set_lineno(). --- Include/internal/pycore_opcode.h | 1 + Lib/opcode.py | 3 + Objects/frameobject.c | 94 +++++++++++++++----------------- Python/bytecodes.c | 3 +- Python/generated_cases.c.h | 10 +++- Python/opcode_metadata.h | 6 +- 6 files changed, 61 insertions(+), 56 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index b8d26b663353b0..a4f1a5ef8e33ef 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -41,6 +41,7 @@ static const uint32_t _PyOpcode_Jump[9] = { }; const uint8_t _PyOpcode_Caches[256] = { + [RESERVED] = 1000000, [BINARY_SUBSCR] = 4, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, diff --git a/Lib/opcode.py b/Lib/opcode.py index 3ca4173e6bd83b..de162cab16ba02 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -444,6 +444,9 @@ def pseudo_op(name, op, real_ops): "SEND": { "counter": 1, }, + "RESERVED": { + "force_failure": 1_000_000 + }, } _inline_cache_entries = [ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7ae56e47200727..86cf579710956d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -128,26 +128,6 @@ frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignore return 0; } -// Given the index of the effective opcode, scan back to construct the oparg -// with EXTENDED_ARG. This only works correctly with *unquickened* code, -// obtained via a call to _PyCode_GetCode! -static unsigned int -get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) -{ - _Py_CODEUNIT word; - unsigned int oparg = codestr[i].op.arg; - if (i >= 1 && (word = codestr[i-1]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 8; - if (i >= 2 && (word = codestr[i-2]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 16; - if (i >= 3 && (word = codestr[i-3]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 24; - } - } - } - return oparg; -} - /* Model the evaluation stack, to determine which jumps * are safe and how many values needs to be popped. * The stack is modelled by a 64 integer, treating any @@ -323,46 +303,52 @@ mark_stacks(PyCodeObject *code_obj, int len) while (todo) { todo = 0; /* Scan instructions */ - for (i = 0; i < len; i++) { + for (i = 0; i < len;) { int64_t next_stack = stacks[i]; + opcode = _Py_GetBaseOpcode(code_obj, i); + int oparg = 0; + while (opcode == EXTENDED_ARG) { + oparg = (oparg << 8) | code[i].op.arg; + i++; + opcode = _Py_GetBaseOpcode(code_obj, i); + stacks[i] = next_stack; + } + int next_i = i + _PyOpcode_Caches[opcode] + 1; if (next_stack == UNINITIALIZED) { + i = next_i; continue; } - opcode = code[i].op.code; + oparg = (oparg << 8) | code[i].op.arg; switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { int64_t target_stack; - int j = get_arg(code, i); - j += i + 1; + int j = next_i + oparg; assert(j < len); - if (stacks[j] == UNINITIALIZED && j < i) { - todo = 1; - } next_stack = pop_value(next_stack); target_stack = next_stack; assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SEND: - j = get_arg(code, i) + i + INLINE_CACHE_ENTRIES_SEND + 1; + j = oparg + i + INLINE_CACHE_ENTRIES_SEND + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case JUMP_FORWARD: - j = get_arg(code, i) + i + 1; + j = oparg + i + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; break; case JUMP_BACKWARD: case JUMP_BACKWARD_NO_INTERRUPT: - j = i + 1 - get_arg(code, i); + j = i + 1 - oparg; assert(j >= 0); assert(j < len); if (stacks[j] == UNINITIALIZED && j < i) { @@ -374,13 +360,13 @@ mark_stacks(PyCodeObject *code_obj, int len) case GET_ITER: case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case FOR_ITER: { int64_t target_stack = push_value(next_stack, Object); - stacks[i+1] = target_stack; - j = get_arg(code, i) + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; + stacks[next_i] = target_stack; + j = oparg + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; @@ -388,16 +374,16 @@ mark_stacks(PyCodeObject *code_obj, int len) } case END_ASYNC_FOR: next_stack = pop_value(pop_value(next_stack)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case PUSH_EXC_INFO: next_stack = push_value(next_stack, Except); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case POP_EXCEPT: assert(top_of_stack(next_stack) == Except); next_stack = pop_value(next_stack); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case RETURN_VALUE: assert(pop_value(next_stack) == EMPTY_STACK); @@ -413,57 +399,62 @@ mark_stacks(PyCodeObject *code_obj, int len) break; case PUSH_NULL: next_stack = push_value(next_stack, Null); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case LOAD_GLOBAL: { - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = push_value(next_stack, Null); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case LOAD_ATTR: { assert(top_of_stack(next_stack) == Object); - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = pop_value(next_stack); next_stack = push_value(next_stack, Null); next_stack = push_value(next_stack, Object); } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case CALL: { - int args = get_arg(code, i); + int args = oparg; for (int j = 0; j < args+2; j++) { next_stack = pop_value(next_stack); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SWAP: { - int n = get_arg(code, i); + int n = oparg; next_stack = stack_swap(next_stack, n); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case COPY: { - int n = get_arg(code, i); + int n = oparg; next_stack = push_value(next_stack, peek(next_stack, n)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } + case CACHE: + case RESERVED: + { + assert(0); + } default: { - int delta = PyCompile_OpcodeStackEffect(opcode, get_arg(code, i)); + int delta = PyCompile_OpcodeStackEffect(opcode, oparg); assert(delta != PY_INVALID_STACK_EFFECT); while (delta < 0) { next_stack = pop_value(next_stack); @@ -473,9 +464,10 @@ mark_stacks(PyCodeObject *code_obj, int len) next_stack = push_value(next_stack, Object); delta--; } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; } } + i = next_i; } /* Scan exception table */ unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code_obj->co_exceptiontable); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6768513f8d4799..6faa1ac3d4f14b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3245,7 +3245,8 @@ dummy_func( Py_UNREACHABLE(); } - inst(RESERVED, (--)) { + inst(RESERVED, (unused1/4, unused2/4, unused3/4, unused4/4, unused5[1000] --)) { + /* In case we haven't crashed already */ assert(0 && "Executing RESERVED instruction."); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1c9f94ef40a2e1..04ada5037d351c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4551,8 +4551,16 @@ } TARGET(RESERVED) { + PyObject **unused5 = (stack_pointer - 1000); + PyObject *unused1 = read_obj(&next_instr[0].cache); + PyObject *unused2 = read_obj(&next_instr[4].cache); + PyObject *unused3 = read_obj(&next_instr[8].cache); + PyObject *unused4 = read_obj(&next_instr[12].cache); #line 3249 "Python/bytecodes.c" + /* In case we haven't crashed already */ assert(0 && "Executing RESERVED instruction."); - #line 4557 "Python/generated_cases.c.h" + #line 4563 "Python/generated_cases.c.h" + STACK_SHRINK(1000); + next_instr += 16; DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 7134bbf8f7ac0f..2c26a9122ee9b1 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -380,7 +380,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CACHE: return 0; case RESERVED: - return 0; + return 1000; default: return -1; } @@ -771,7 +771,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000, INSTR_FMT_IXC000000000000000 }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -966,6 +966,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, [EXTENDED_ARG] = { true, INSTR_FMT_IB }, [CACHE] = { true, INSTR_FMT_IX }, - [RESERVED] = { true, INSTR_FMT_IX }, + [RESERVED] = { true, INSTR_FMT_IXC000000000000000 }, }; #endif From 65c548e8776261f05ec46661da312490ae7dbbd1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:02:59 +0000 Subject: [PATCH 075/116] Make line instrumentation able to handle multiple inits. --- Objects/codeobject.c | 9 +-- Python/instrumentation.c | 146 +++++++++++++-------------------------- Python/specialize.c | 3 +- 3 files changed, 54 insertions(+), 104 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 024bd7bf78daa5..6288334eba11da 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1487,13 +1487,10 @@ deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; - if (caches) { - instructions[i+1].cache = adaptive_counter_warmup(); - for (int j = 2; j <= caches; j++) { - instructions[i+j].cache = 0; - } - i += caches; + for (int j = 1; j <= caches; j++) { + instructions[i+j].cache = 0; } + i += caches; } } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 59f500c14570e4..9168df3e3c3d2a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -268,28 +268,6 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } -/* We need to parse instructions from code objects a lot, - * so add a couple of utilities */ -int _Py_GetBaseOpcode(PyCodeObject *code, int offset) -{ - int opcode = _PyCode_CODE(code)[offset].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - assert(opcode != INSTRUMENTED_INSTRUCTION); - assert(opcode != INSTRUMENTED_LINE); - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - assert(_PyOpcode_Deopt[opcode] != 0); - assert(opcode != RESERVED); - return _PyOpcode_Deopt[opcode]; -} - int instruction_length(PyCodeObject *code, int offset) { int opcode = _PyCode_CODE(code)[offset].op.code; @@ -325,38 +303,6 @@ typedef struct _Instruction { bool is_specialized_instrumented; } Instruction; -static Instruction -read_instruction(PyCodeObject *code, int offset) -{ - Instruction result = (Instruction){0}; - int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - result.is_prefix_instrumented = true; - } - - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - result.is_prefix_instrumented = true; - } - while (opcode == EXTENDED_ARG) { - result.extended_args += 1; - offset++; - opcode = _PyCode_CODE(code)[offset].op.code; - } - result.length = result.extended_args + 1; - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - result.deinstrumented_opcode = deinstrumented; - result.is_specialized_instrumented = true; - } - else { - result.deinstrumented_opcode = opcode; - } - result.length = result.extended_args + instruction_length(code, offset); - return result; -} - #ifdef INSTRUMENT_DEBUG static void @@ -379,9 +325,6 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData else if (lines[i].original_opcode == 0) { fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta); } - else if (lines[i].original_opcode == 255) { - fprintf(out, ", lines = {original_opcode = LINE_MARKER, line_delta = %d)", lines[i].line_delta); - } else { fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta); } @@ -508,7 +451,9 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } while (0) bool valid_opcode(int opcode) { - if (opcode > 0 && opcode < 255 && + if (opcode > 0 && + opcode != RESERVED && + opcode < 255 && _PyOpcode_OpName[opcode] && _PyOpcode_OpName[opcode][0] != '<' ) { @@ -535,8 +480,8 @@ sanity_check_instrumentation(PyCodeObject *code) ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len;) { - Instruction inst = read_instruction(code, i); - int opcode = inst.actual_opcode; + int opcode = _PyCode_CODE(code)[i].op.code; + int base_opcode = _Py_GetBaseOpcode(code, i); if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = data->per_instruction_opcodes[i]; if (!is_instrumented(opcode)) { @@ -547,7 +492,7 @@ sanity_check_instrumentation(PyCodeObject *code) if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); - int opcode = data->lines[i].original_opcode; + opcode = data->lines[i].original_opcode; CHECK(opcode != END_FOR); CHECK(opcode != RESUME); CHECK(opcode != INSTRUMENTED_RESUME); @@ -556,38 +501,32 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(opcode != INSTRUMENTED_LINE); } - else if (data->lines) { + else if (data->lines && !is_instrumented(opcode)) { CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == 255); + data->lines[i].original_opcode == base_opcode); } - if (inst.is_specialized_instrumented) { - int event = EVENT_FOR_OPCODE[inst.deinstrumented_opcode]; + if (is_instrumented(opcode)) { + CHECK(DE_INSTRUMENT[opcode] == base_opcode); + int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]]; if (event < 0) { /* RESUME fixup */ - event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; + event = _PyCode_CODE(code)[i].op.arg; } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && inst.deinstrumented_opcode != END_FOR) { + if (data->lines && base_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } - CHECK(valid_opcode(inst.deinstrumented_opcode)); - if (inst.is_prefix_instrumented) { - CHECK(inst.extended_args || _PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); - } - if (inst.is_specialized_instrumented) { - CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); - } + CHECK(valid_opcode(opcode)); if (data->tools) { - uint8_t local_tools = data->tools[i + inst.extended_args]; - int deopt = _PyOpcode_Deopt[inst.deinstrumented_opcode]; - if (OPCODE_HAS_EVENT[deopt]) { - int event = EVENT_FOR_OPCODE[deopt]; + uint8_t local_tools = data->tools[i]; + if (OPCODE_HAS_EVENT[base_opcode]) { + int event = EVENT_FOR_OPCODE[base_opcode]; if (event == -1) { /* RESUME fixup */ - event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; + event = _PyCode_CODE(code)[i].op.arg; } CHECK((active_monitors.tools[event] & local_tools) == local_tools); } @@ -595,7 +534,7 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(local_tools == 0xff); } } - i += inst.length; + i += instruction_length(code, i); assert(i <= code_len); } } @@ -605,6 +544,27 @@ sanity_check_instrumentation(PyCodeObject *code) #endif +/* Get the underlying opcode, stripping instrumentation */ +int _Py_GetBaseOpcode(PyCodeObject *code, int i) +{ + int opcode = _PyCode_CODE(code)[i].op.code; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[i].original_opcode; + } + CHECK(opcode != INSTRUMENTED_INSTRUCTION); + CHECK(opcode != INSTRUMENTED_LINE); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + CHECK(_PyOpcode_Deopt[opcode] != 0); + CHECK(opcode != RESERVED); + return _PyOpcode_Deopt[opcode]; +} + static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -682,8 +642,6 @@ de_instrument_line(PyCodeObject *code, int i) instr->op.code = original_opcode; } assert(instr->op.code != INSTRUMENTED_LINE); - /* Mark instruction as candidate for line instrumentation */ - lines->original_opcode = 255; } @@ -752,19 +710,15 @@ instrument_line(PyCodeObject *code, int i) return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - CHECK(lines->original_opcode == 255); - if (is_instrumented(opcode)) { - lines->original_opcode = opcode; - } - else { - assert(opcode != 0); - assert(_PyOpcode_Deopt[opcode] != RESUME); - lines->original_opcode = _PyOpcode_Deopt[opcode]; - CHECK(lines->original_opcode != 0); - } - assert(lines->original_opcode > 0); + CHECK(_PyOpcode_Deopt[opcode] == lines->original_opcode || + ( + is_instrumented(opcode) && + lines->original_opcode == DE_INSTRUMENT[opcode] + ) + ); + lines->original_opcode = _PyOpcode_Deopt[opcode]; + CHECK(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; - CHECK(code->_co_monitoring->lines[i].original_opcode != 0); } static void @@ -1343,8 +1297,6 @@ initialize_tools(PyCodeObject *code) } } -#define LINE_MARKER 255 - #define NO_LINE -128 static void @@ -1377,7 +1329,7 @@ initialize_lines(PyCodeObject *code) break; default: if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; + line_data[i].original_opcode = opcode; } else { line_data[i].original_opcode = 0; diff --git a/Python/specialize.c b/Python/specialize.c index 38392579be08b5..77e636d3ac2d2b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -273,7 +273,8 @@ _PyCode_Quicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; - opcode = _PyOpcode_Deopt[instructions[i].op.code]; + opcode = _Py_GetBaseOpcode(code, i); + assert(opcode < 230); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From 575f7d15c8377691448de359bb8e790ce43950a8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:14:13 +0000 Subject: [PATCH 076/116] Add brief comments on instrumentation data structures. --- Include/cpython/code.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 39ec413c9d4980..c61926b6fd19b2 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -64,11 +64,17 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +/* Ancilliary data structure used for instrumentation. + Line instrumentation creates an array of + these. One entry per code unit.*/ typedef struct { uint8_t original_opcode; int8_t line_delta; } _PyCoLineInstrumentationData; +/* Main data structure used for instrumentation. + * This is allocated when needed for instrumentation + */ typedef struct { _Py_Monitors local_monitors; _Py_Monitors active_monitors; From 415741d30fddb7b1c99b6b082c4f60eb81c959d4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:23:35 +0000 Subject: [PATCH 077/116] Add symbol constant for min instrumented opcode. --- Include/cpython/code.h | 1 + Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test__opcode.py | 6 ++---- Programs/test_frozenmain.h | 16 ++++++++-------- Tools/build/generate_opcode_h.py | 3 +++ 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index c61926b6fd19b2..ea2d9a3d72f1a5 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -11,6 +11,7 @@ extern "C" { #define PY_MONITORING_EVENTS 16 #define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Table of which tools are active for each monitored event. */ typedef struct _Py_Monitors { uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; } _Py_Monitors; diff --git a/Include/opcode.h b/Include/opcode.h index 701d7f81837e5e..aa8716ef5b4030 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 +#define MIN_INSTRUMENTED_OPCODE 238 #define INSTRUMENTED_POP_JUMP_IF_NONE 238 #define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 #define INSTRUMENTED_RESUME 240 diff --git a/Lib/opcode.py b/Lib/opcode.py index de162cab16ba02..f471dd9c65f0d2 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -225,6 +225,7 @@ def pseudo_op(name, op, real_ops): def_op('CALL_INTRINSIC_2', 174) # Instrumented instructions +MIN_INSTRUMENTED_OPCODE = 238 def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index eb8666cc41ce8b..7640c6fb57d4f3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,8 +20,7 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 230: - # Opcodes 230 and up are internal for instrumentation + if code >= opcode.MIN_INSTRUMENTED_OPCODE: continue with self.subTest(opname=name): if code not in has_arg: @@ -50,8 +49,7 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 230: - # Opcodes 230 and up are internal for instrumentation + if code >= opcode.MIN_INSTRUMENTED_OPCODE: continue with self.subTest(opname=name): if code not in has_arg: diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 57635a398d1df0..8e5055bd7bceb1 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,16 +3,16 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,182,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,17,0,0,0,0,0,0,0,1,0,2,0, - 101,2,100,3,101,0,106,6,17,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,17,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,17,0,0,0, + 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, + 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, + 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 17,0,0,0,0,0,0,0,100,4,25,0,17,0,0,0, - 0,0,0,0,90,5,100,5,68,0,93,23,17,0,90,6, + 0,0,0,0,0,0,0,0,100,4,25,0,0,0,0,0, + 0,0,0,0,90,5,100,5,68,0,93,23,0,0,90,6, 2,0,101,2,100,6,101,6,155,0,100,7,101,5,101,6, - 25,0,17,0,0,0,0,0,0,0,155,0,157,4,171,1, - 17,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, + 25,0,0,0,0,0,0,0,0,0,155,0,157,4,171,1, + 0,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 7aaef6b16f3443..53cf11ddeb0093 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -89,6 +89,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna HAVE_ARGUMENT = opcode["HAVE_ARGUMENT"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] + MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) used = [ False ] * len(opname) @@ -117,6 +118,8 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna fobj.write(DEFINE.format("HAVE_ARGUMENT", HAVE_ARGUMENT)) if op == MIN_PSEUDO_OPCODE: fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) + if op == MIN_INSTRUMENTED_OPCODE: + fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) From d70a1a417d3a8bb44a15f04e03493e9148305527 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:44:26 +0000 Subject: [PATCH 078/116] Code cleanup from review. --- Python/instrumentation.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9168df3e3c3d2a..63dbb2c7908ec9 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -145,7 +145,7 @@ static inline bool is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return INSTRUMENTED_OPCODES[opcode] == opcode; + return opcode >= MIN_INSTRUMENTED_OPCODE; } static inline bool @@ -190,7 +190,7 @@ monitors_or(_Py_Monitors a, _Py_Monitors b) } static inline bool -monitors_empty(_Py_Monitors m) +monitors_are_empty(_Py_Monitors m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { @@ -200,7 +200,7 @@ monitors_empty(_Py_Monitors m) return true; } -static inline int +static inline bool multiple_tools(_Py_Monitors *m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -227,7 +227,7 @@ get_events(_Py_Monitors *m, int tool_id) * 8 bit value. * if line_delta == -128: * line = None # represented as -1 - * elif line == -127: + * elif line_delta == -127: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); * else: * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; @@ -241,9 +241,6 @@ compute_line_delta(PyCodeObject *code, int offset, int line) if (line < 0) { return -128; } - // assert(line >= code->co_firstlineno); - // assert(offset >= code->_co_firsttraceable); - // assert(offset < Py_SIZE(code)); int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); if (delta < 128 && delta > -128) { return delta; @@ -255,7 +252,6 @@ static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { if (line_delta > -127) { - // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; } if (line_delta == -128) { @@ -482,6 +478,8 @@ sanity_check_instrumentation(PyCodeObject *code) for (int i = 0; i < code_len;) { int opcode = _PyCode_CODE(code)[i].op.code; int base_opcode = _Py_GetBaseOpcode(code, i); + CHECK(valid_opcode(opcode)); + CHECK(valid_opcode(base_opcode)); if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = data->per_instruction_opcodes[i]; if (!is_instrumented(opcode)) { @@ -503,7 +501,8 @@ sanity_check_instrumentation(PyCodeObject *code) } else if (data->lines && !is_instrumented(opcode)) { CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == base_opcode); + data->lines[i].original_opcode == base_opcode || + DE_INSTRUMENT[data->lines[i].original_opcode] == base_opcode); } if (is_instrumented(opcode)) { CHECK(DE_INSTRUMENT[opcode] == base_opcode); @@ -1493,11 +1492,11 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) else { removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events); new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors); - assert(monitors_empty(monitors_and(new_events, removed_events))); + assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; code->_co_instrumentation_version = interp->monitoring_version; - if (monitors_empty(new_events) && monitors_empty(removed_events)) { + if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif From d580de6338dcea14374b74a16ad72b43f34c5e29 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 16:06:35 +0000 Subject: [PATCH 079/116] A bit more cleanup. --- Python/bytecodes.c | 5 ++-- Python/instrumentation.c | 49 ++++++++-------------------------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6faa1ac3d4f14b..7426c1d8cfa1b4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3242,12 +3242,13 @@ dummy_func( } inst(CACHE, (--)) { + assert(0 && "Executing a cache."); Py_UNREACHABLE(); } - inst(RESERVED, (unused1/4, unused2/4, unused3/4, unused4/4, unused5[1000] --)) { - /* In case we haven't crashed already */ + inst(RESERVED, (--)) { assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 63dbb2c7908ec9..6a02c9319385f6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -57,38 +57,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, }; -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_CONST] = true, - [INSTRUMENTED_RETURN_CONST] = true, - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_JUMP_FORWARD] = true, - [INSTRUMENTED_JUMP_BACKWARD] = true, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, - [INSTRUMENTED_POP_JUMP_IF_NONE] = true, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_FOR_ITER] = true, - [END_FOR] = true, - [INSTRUMENTED_END_FOR] = true, - [END_SEND] = true, - [INSTRUMENTED_END_SEND] = true, -}; - static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, @@ -141,6 +109,12 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +static inline bool +opcode_has_event(int opcode) { + return opcode < INSTRUMENTED_LINE && + INSTRUMENTED_OPCODES[opcode] > 0; +} + static inline bool is_instrumented(int opcode) { assert(opcode != 0); @@ -559,8 +533,6 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i) if (deinstrumented) { return deinstrumented; } - CHECK(_PyOpcode_Deopt[opcode] != 0); - CHECK(opcode != RESERVED); return _PyOpcode_Deopt[opcode]; } @@ -754,8 +726,7 @@ instruction_has_event(PyCodeObject *code, int offset) if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } - assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); - return OPCODE_HAS_EVENT[opcode]; + return opcode_has_event(opcode); } #endif @@ -1264,7 +1235,7 @@ initialize_tools(PyCodeObject *code) assert(opcode != 0); } opcode = _PyOpcode_Deopt[opcode]; - if (OPCODE_HAS_EVENT[opcode]) { + if (opcode_has_event(opcode)) { if (instrumented) { int8_t event; if (opcode == RESUME) { @@ -1510,7 +1481,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); - if (OPCODE_HAS_EVENT[base_opcode]) { + if (opcode_has_event(base_opcode)) { int8_t event; if (base_opcode == RESUME) { event = instr->op.arg > 0; From 2076d5fe143c89368e0f4fe21a6f27e4f3560b92 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:01:21 +0000 Subject: [PATCH 080/116] Add a few more comments. --- Include/cpython/code.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ea2d9a3d72f1a5..bf7a3b666b2d39 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -77,12 +77,20 @@ typedef struct { * This is allocated when needed for instrumentation */ typedef struct { + /* Monitoring specific to this code object */ _Py_Monitors local_monitors; + /* Monitoring that is active on this code object */ _Py_Monitors active_monitors; + /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* Information to support line events */ _PyCoLineInstrumentationData *lines; + /* The tools that are to be notified for line events for the matching code unit */ uint8_t *line_tools; + /* Information to support instruction events */ + /* The underlying instructions, which can themselves be instrumented */ uint8_t *per_instruction_opcodes; + /* The tools that are to be notified for instruction events for the matching code unit */ uint8_t *per_instruction_tools; } _PyCoMonitoringData; From 7b32d7974559f8d6188c43d413e30b78a5427335 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:04:10 +0000 Subject: [PATCH 081/116] Remove unused code. --- Include/internal/pycore_instruments.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index f34bd074e6a92f..e3327c29389dd2 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -43,14 +43,10 @@ extern "C" { #define PY_MONITORING_EVENT_C_RAISE 15 -/* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ - - typedef uint32_t _PyMonitoringEventSet; /* Reserved IDs */ -#define PY_INSTRUMENT_PEP_523 5 #define PY_INSTRUMENT_SYS_PROFILE 6 #define PY_INSTRUMENT_SYS_TRACE 7 From 662c16c6911793b2413f1e8ce03425a3af31fb64 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:17:46 +0000 Subject: [PATCH 082/116] Fix leak of monitoring blocks. --- Objects/codeobject.c | 62 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 6288334eba11da..34e1edc6c2678b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1642,6 +1642,30 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, return co; } +static void +free_monitoring_data(_PyCoMonitoringData *data) +{ + if (data == NULL) { + return; + } + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + PyMem_Free(data); +} + static void code_dealloc(PyCodeObject *co) { @@ -1688,24 +1712,7 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } - _PyCoMonitoringData *data = co->_co_monitoring; - if (data) { - if (data->tools) { - PyMem_Free(data->tools); - } - if (data->lines) { - PyMem_Free(data->lines); - } - if (data->line_tools) { - PyMem_Free(data->line_tools); - } - if (data->per_instruction_opcodes) { - PyMem_Free(data->per_instruction_opcodes); - } - if (data->per_instruction_tools) { - PyMem_Free(data->per_instruction_tools); - } - } + free_monitoring_data(co->_co_monitoring); PyObject_Free(co); } @@ -2288,24 +2295,7 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } - _PyCoMonitoringData *data = co->_co_monitoring; - if (data) { - if (data->tools) { - PyMem_Free(data->tools); - } - if (data->lines) { - PyMem_Free(data->lines); - } - if (data->line_tools) { - PyMem_Free(data->line_tools); - } - if (data->per_instruction_opcodes) { - PyMem_Free(data->per_instruction_opcodes); - } - if (data->per_instruction_tools) { - PyMem_Free(data->per_instruction_tools); - } - } + free_monitoring_data(co->_co_monitoring); } int From 51a93e7ff64c4541acd71c8331a2b26e46cbcd2c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 19:02:39 +0000 Subject: [PATCH 083/116] Make function static. --- Python/instrumentation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6a02c9319385f6..869b4445344151 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -238,7 +238,8 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } -int instruction_length(PyCodeObject *code, int offset) +static int +instruction_length(PyCodeObject *code, int offset) { int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); From 64bf37ff351046b2ab5ed4d528d917ac9b0fb194 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 07:47:22 +0100 Subject: [PATCH 084/116] Use symbolic constants in line computations. --- Python/instrumentation.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 869b4445344151..4736c3ed1e9b6d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -207,35 +207,37 @@ get_events(_Py_Monitors *m, int tool_id) * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; */ +#define NO_LINE -128 +#define COMPUTED_LINE -127 + #define OFFSET_SHIFT 4 static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { if (line < 0) { - return -128; + return NO_LINE; } int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); - if (delta < 128 && delta > -128) { + if (delta <= INT8_MAX && delta > COMPUTED_LINE) { return delta; } - return -127; + return COMPUTED_LINE; } static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - if (line_delta > -127) { + if (line_delta > COMPUTED_LINE) { return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; } - if (line_delta == -128) { + if (line_delta == NO_LINE) { + return -1; } - else { - assert(line_delta == -127); - /* Look it up */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } + assert(line_delta == COMPUTED_LINE); + /* Look it up */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } static int From 5faec77e8e8e5407e1202019dfa7835d5da49d30 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 07:53:05 +0100 Subject: [PATCH 085/116] Make data static. --- Python/instrumentation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4736c3ed1e9b6d..bd68901a72ba23 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1908,7 +1908,7 @@ add_power2_constant(PyObject *obj, const char *name, int i) return err; } -const char *const event_names [] = { +static const char *const event_names [] = { [PY_MONITORING_EVENT_PY_START] = "PY_START", [PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME", [PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN", From 0be156286aa08b30205c24dbaf067e6c29355087 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 08:12:15 +0100 Subject: [PATCH 086/116] Add extra check. --- Python/instrumentation.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index bd68901a72ba23..66f485b2e65723 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -// #define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -401,7 +401,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); dump_instrumentation_data_per_instruction(code, data, i, out); - /* TO DO -- per instruction data */ fprintf(out, "\n"); ; } @@ -462,7 +461,10 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - /* TO DO -- check tools */ + if (data->per_instruction_tools) { + uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; + CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); + } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); @@ -498,7 +500,7 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(valid_opcode(opcode)); if (data->tools) { uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[base_opcode]) { + if (opcode_has_event(base_opcode)) { int event = EVENT_FOR_OPCODE[base_opcode]; if (event == -1) { /* RESUME fixup */ From b32a075094f175ad30fef0939ac48016ec34f3fd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 10:38:44 +0100 Subject: [PATCH 087/116] Make sure that instrumentation of line and instructions at the same time is supported. --- Lib/test/test_monitoring.py | 162 ++++++++++++++++++++++++++++++++++++ Python/instrumentation.c | 38 +++++---- 2 files changed, 183 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 9278a474ed66fe..0b1e171ac3a474 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -767,6 +767,168 @@ def func3(): ('line', 'check_events', 11), ('call', 'set_events', 2)]) +class InstructionRecorder: + + event_type = E.INSTRUCTION + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset): + # Filter out instructions in check_events to lower noise + if code.co_name != "check_events": + self.events.append(("instruction", code.co_name, offset)) + + +LINE_AND_INSTRUCTION_RECORDERS = InstructionRecorder, LineRecorder + +class TestLineAndInstructionEvents(CheckEvents): + maxDiff = None + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func1', 1), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 3), + ('line', 'func1', 2), + ('instruction', 'func1', 4), + ('instruction', 'func1', 5), + ('line', 'func1', 3), + ('instruction', 'func1', 6), + ('instruction', 'func1', 7), + ('line', 'check_events', 11)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func2', 1), + ('line', 'func2', 1), + ('instruction', 'func2', 2), + ('instruction', 'func2', 3), + ('line', 'func2', 2), + ('instruction', 'func2', 4), + ('instruction', 'func2', 14), + ('instruction', 'func2', 15), + ('instruction', 'func2', 20), + ('instruction', 'func2', 21), + ('line', 'func2', 3), + ('instruction', 'func2', 22), + ('instruction', 'func2', 23), + ('line', 'check_events', 11)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func3', 1), + ('line', 'func3', 1), + ('instruction', 'func3', 2), + ('line', 'func3', 2), + ('instruction', 'func3', 3), + ('instruction', 'func3', 4), + ('line', 'func3', 3), + ('instruction', 'func3', 9), + ('instruction', 'func3', 10), + ('instruction', 'func3', 11), + ('line', 'func3', 4), + ('instruction', 'func3', 12), + ('line', 'func3', 5), + ('instruction', 'func3', 13), + ('instruction', 'func3', 14), + ('instruction', 'func3', 15), + ('line', 'func3', 6), + ('instruction', 'func3', 16), + ('instruction', 'func3', 17), + ('line', 'check_events', 11)]) + +class TestInstallIncrementallly(unittest.TestCase): + + def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + all_events |= recorder.event_type + sys.monitoring.set_events(tool, all_events) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, recorder(event_list)) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + for line in must_include: + self.assertIn(line, event_list) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + @staticmethod + def func1(): + line1 = 1 + + MUST_INCLUDE_LI = [ + ('instruction', 'func1', 1), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 3)] + + def test_line_then_instruction(self): + recorders = [ LineRecorder, InstructionRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, LineRecorderLowNoise ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + @staticmethod + def func2(): + len(()) + + MUST_INCLUDE_CI = [ + ('instruction', 'func2', 1), + ('call', 'func2', sys.monitoring.MISSING), + ('call', 'len', ()), + ('instruction', 'func2', 6), + ('instruction', 'func2', 7)] + + + + def test_line_then_instruction(self): + recorders = [ CallRecorder, InstructionRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, CallRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + class TestLocalEvents(unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 66f485b2e65723..217a2d62a62af5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -652,24 +652,26 @@ static void instrument(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - CHECK(opcode != INSTRUMENTED_INSTRUCTION); + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode = lines->original_opcode; - CHECK(opcode != 0); + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); - lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; - CHECK(lines->original_opcode != 0); } - else if (!is_instrumented(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - int instrumented = INSTRUMENTED_OPCODES[opcode]; + CHECK(opcode != 0); + if (!is_instrumented(opcode)) { + int deopt = _PyOpcode_Deopt[opcode]; + int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - instr->op.code = instrumented; - if (_PyOpcode_Caches[opcode]) { + *opcode_ptr = instrumented; + if (_PyOpcode_Caches[deopt]) { instr[1].cache = adaptive_counter_warmup(); } } @@ -678,10 +680,12 @@ instrument(PyCodeObject *code, int i) static void instrument_line(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - CHECK(opcode != INSTRUMENTED_INSTRUCTION); + uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_LINE) { return; } @@ -694,7 +698,7 @@ instrument_line(PyCodeObject *code, int i) ); lines->original_opcode = _PyOpcode_Deopt[opcode]; CHECK(lines->original_opcode > 0); - instr->op.code = INSTRUMENTED_LINE; + *opcode_ptr = INSTRUMENTED_LINE; } static void From 85d6923432491583bd70628c6340be8d11852acb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 15:31:48 +0100 Subject: [PATCH 088/116] Remove incorrect assertion. --- Python/instrumentation.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 217a2d62a62af5..9ca745d2f65c85 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -690,12 +690,6 @@ instrument_line(PyCodeObject *code, int i) return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - CHECK(_PyOpcode_Deopt[opcode] == lines->original_opcode || - ( - is_instrumented(opcode) && - lines->original_opcode == DE_INSTRUMENT[opcode] - ) - ); lines->original_opcode = _PyOpcode_Deopt[opcode]; CHECK(lines->original_opcode > 0); *opcode_ptr = INSTRUMENTED_LINE; From ebcc42f134d1f6a44c2891cd48bf25d64d5c184f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 16:03:27 +0100 Subject: [PATCH 089/116] Turn off instrumentation debugging. --- Python/instrumentation.c | 98 ++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 65 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9ca745d2f65c85..2ff4f35218fb7e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -542,47 +542,29 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i) } static void -de_instrument(PyCodeObject *code, int offset, int event) +de_instrument(PyCodeObject *code, int i, int event) { assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; - int opcode = _Py_OPCODE(*instr); - if (!is_instrumented(opcode)) { - return; + + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode = *opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; } - switch(opcode) { - case INSTRUMENTED_INSTRUCTION: - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - if (opcode != INSTRUMENTED_LINE) { - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - code->_co_monitoring->per_instruction_opcodes[offset] = opcode = deinstrumented; - } - break; - } - /* Intentional fall through */ - case INSTRUMENTED_LINE: - { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; - int original_opcode = lines->original_opcode; - int deinstrumented = DE_INSTRUMENT[original_opcode]; - if (deinstrumented) { - lines->original_opcode = opcode = deinstrumented; - } - break; - } - default: - { - int deinstrumented = DE_INSTRUMENT[opcode]; - assert(deinstrumented); - instr->op.code = opcode = deinstrumented; - } + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; } - assert(!is_instrumented(opcode)); - int base_opcode = _PyOpcode_Deopt[opcode]; - assert(base_opcode != 0); - if (_PyOpcode_Caches[base_opcode]) { + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented == 0) { + return; + } + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + *opcode_ptr = deinstrumented; + if (_PyOpcode_Caches[deinstrumented]) { instr[1].cache = adaptive_counter_warmup(); } } @@ -591,32 +573,24 @@ static void de_instrument_line(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; } if (opcode != INSTRUMENTED_LINE) { return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; - if (is_instrumented(original_opcode)) { - /* Instrumented original */ - instr->op.code = original_opcode; - } - else { - CHECK(original_opcode != 0); - CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); - } - } - if (instr->op.code == INSTRUMENTED_INSTRUCTION) { - code->_co_monitoring->per_instruction_opcodes[i] = original_opcode; - } - else { - instr->op.code = original_opcode; + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + *opcode_ptr = instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); } + assert(*opcode_ptr != INSTRUMENTED_LINE); assert(instr->op.code != INSTRUMENTED_LINE); } @@ -631,16 +605,10 @@ de_instrument_per_instruction(PyCodeObject *code, int i) } int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; CHECK(original_opcode != 0); - if (is_instrumented(original_opcode)) { - /* Instrumented original */ - instr->op.code = original_opcode; - } - else { - CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - instr->op.code = original_opcode; - if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); - } + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for sanity check */ From 7cfbc7ee4ff581727549868f80d4e9fcc52f458e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 16:35:04 +0100 Subject: [PATCH 090/116] Fix a couple of errors. --- Lib/test/test_monitoring.py | 8 ++++---- Objects/codeobject.c | 1 + Python/instrumentation.c | 6 +++++- Python/pystate.c | 3 +++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0b1e171ac3a474..8cc2fa8792f4fa 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -480,7 +480,7 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() -class LineMontoringTest(unittest.TestCase): +class LineMonitoringTest(unittest.TestCase): def test_lines_single(self): try: @@ -492,7 +492,7 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMontoringTest.test_lines_single.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno self.assertEqual(events, [start+7, 14, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMontoringTest.test_lines_loop.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) @@ -532,7 +532,7 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMontoringTest.test_lines_two.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno expected = [start+10, 14, start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 34e1edc6c2678b..693ab710c22200 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2296,6 +2296,7 @@ _PyStaticCode_Fini(PyCodeObject *co) co->co_weakreflist = NULL; } free_monitoring_data(co->_co_monitoring); + co->_co_monitoring = NULL; } int diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 2ff4f35218fb7e..0255fb6b9f0875 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1705,6 +1705,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } if (_Py_popcount32(event) != 1) { PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time"); + return NULL; } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { @@ -1913,7 +1914,10 @@ monitoring__all_events_impl(PyObject *module) if (tools == 0) { continue; } - int err = PyDict_SetItemString(res, event_names[e], PyLong_FromLong(tools)); + PyObject *tools_obj = PyLong_FromLong(tools); + assert(tools_obj != NULL); + int err = PyDict_SetItemString(res, event_names[e], tools_obj); + Py_DECREF(tools_obj); if (err < 0) { Py_DECREF(res); return NULL; diff --git a/Python/pystate.c b/Python/pystate.c index 36adfacb3f82c5..822627d8bb3b34 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,6 +815,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->monitoring_tool_names[t]); + } PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); From d366364642405ccc030c0e20780b81f566eea9e6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 17:24:15 +0100 Subject: [PATCH 091/116] Add minimal news entry. --- .../2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst new file mode 100644 index 00000000000000..4fd6fa29bc3648 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst @@ -0,0 +1 @@ +Adds low overhead monitoring support. See PEP 669 for full details. From fdb486026511eda234537e9d3d21a25f6007547b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 17:33:22 +0100 Subject: [PATCH 092/116] Pacify the global object checker. --- Python/instrumentation.c | 14 +++++++------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0255fb6b9f0875..07546417c15aca 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1391,18 +1391,18 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -static const bool super_instructions[256] = { - [LOAD_FAST__LOAD_FAST] = true, - [LOAD_FAST__LOAD_CONST] = true, - [STORE_FAST__LOAD_FAST] = true, - [STORE_FAST__STORE_FAST] = true, - [LOAD_CONST__LOAD_FAST] = true, +static const uint8_t super_instructions[256] = { + [LOAD_FAST__LOAD_FAST] = 1, + [LOAD_FAST__LOAD_CONST] = 1, + [STORE_FAST__LOAD_FAST] = 1, + [STORE_FAST__STORE_FAST] = 1, + [LOAD_CONST__LOAD_FAST] = 1, }; /* Should use instruction metadata for this */ static bool is_super_instruction(int opcode) { - return super_instructions[opcode]; + return super_instructions[opcode] != 0; } int diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e0e45265209fff..f2f7b9b7020bc7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -135,6 +135,8 @@ Objects/stringlib/unicode_format.h - PyFieldNameIter_Type - Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - +Python/legacy_tracing.c - _PyLegacyEventHandler_Type - + ##----------------------- ## static builtin structseq @@ -297,6 +299,8 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Python/instrumentation.c - DISABLE - +Python/instrumentation.c - _PyInstrumentation_MISSING - ################################## From aee722f0a1a469057e76f3ac7c9044959530c322 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 10:15:04 +0100 Subject: [PATCH 093/116] Minor fixups. --- Python/instrumentation.c | 3 ++- Python/pystate.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 07546417c15aca..305cd178c81bc5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1626,7 +1626,8 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) return NULL; } if (!PyUnicode_Check(name)) { - PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); if (interp->monitoring_tool_names[tool_id] != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index 822627d8bb3b34..5b2dc5a97a79a8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -670,6 +670,8 @@ init_interpreter(PyInterpreterState *interp, } } + interp->sys_profile_initialized = false; + interp->sys_profile_initialized = false; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -815,6 +817,8 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } + interp->sys_profile_initialized = false; + interp->sys_profile_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } From bba53b8198491d161ce061da6fe953484b2286db Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 10:39:07 +0100 Subject: [PATCH 094/116] Fix refleak when error in RETURN instrumentation. --- Include/internal/pycore_instruments.h | 2 -- Python/bytecodes.c | 6 +++--- Python/generated_cases.c.h | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e3327c29389dd2..e4c262c9e213a9 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -51,8 +51,6 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_INSTRUMENT_SYS_TRACE 7 -/* API functions */ - PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be98fa22b86222..4bb51bd51fb912 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -597,7 +597,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -628,11 +628,11 @@ dummy_func( inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(retval); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; + Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 89d865b50e1617..8f8b6ae50f3c0b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -931,7 +931,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - if (err) goto pop_1_error; + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -966,11 +966,11 @@ TARGET(INSTRUMENTED_RETURN_CONST) { #line 630 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(retval); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); if (err) goto error; + Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); From 97ec1c5498b54e4aaaf09de5f407af443271f14c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 12:20:55 +0100 Subject: [PATCH 095/116] Fix another refleak in error handling in instrumented instruction. --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fd528db85f723b..8a5649503137cd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -877,7 +877,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 81eb702554e4c6..054efa24d21082 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1206,7 +1206,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, retval); - if (err) goto pop_1_error; + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); From edc6709d6f687f418876a7fe1e2b15acc8218fdf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 14:18:18 +0100 Subject: [PATCH 096/116] Remove debug code ready for final review. --- Include/internal/pycore_opcode.h | 1 - Lib/importlib/_bootstrap_external.py | 5 ++--- Lib/opcode.py | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 65588bc01fd739..a79afda558dc92 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -45,7 +45,6 @@ const uint32_t _PyOpcode_Jump[9] = { }; const uint8_t _PyOpcode_Caches[256] = { - [RESERVED] = 1000000, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 8cf7d374d1a420..8a52946de690b4 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -438,8 +438,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) - -# Python 3.12a? 3526 (Add instrumentation support) +# Python 3.12a7 3525 (Add instrumentation support) # Python 3.13 will start with 3550 @@ -456,7 +455,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index d69bc7091450f7..5aa32d3cb77613 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -443,9 +443,6 @@ def pseudo_op(name, op, real_ops): "SEND": { "counter": 1, }, - "RESERVED": { - "force_failure": 1_000_000 - }, } _inline_cache_entries = [ From e40a68f367c73685247fe066a3d1b76c602c8b00 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 15:27:58 +0100 Subject: [PATCH 097/116] Fix another (and hopefully final) refleak when handling an error in an instrumented bytecode. --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a5649503137cd..39f5add0019989 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3027,7 +3027,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, func, arg); - ERROR_IF(err, error); + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 054efa24d21082..cc108f01a2062d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4193,7 +4193,7 @@ int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, func, arg); - if (err) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( From d9f8192a6549ec60a25e2858257925f23614ffcc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:15:46 +0100 Subject: [PATCH 098/116] Comment and rename #defines to better reflect names in PEP. --- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_instruments.h | 14 +++++----- Python/bytecodes.c | 4 +-- Python/ceval.c | 4 +-- Python/generated_cases.c.h | 4 +-- Python/instrumentation.c | 2 +- Python/legacy_tracing.c | 40 +++++++++++++-------------- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 93b37e6fee64b3..acdfe17ee7cd99 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -143,7 +143,7 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) values are not visible to the cycle GC. We choose -1 rather than 0 to assist debugging. */ static inline PyObject** -_PyFrame_FetchStackPointer(_PyInterpreterFrame *frame) +_PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { PyObject **sp = frame->localsplus+frame->stacktop; frame->stacktop = -1; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e4c262c9e213a9..9615efcaf0a35b 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -14,7 +14,8 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Require bytecode instrumentation */ +/* Local events. + * These require bytecode instrumentation */ #define PY_MONITORING_EVENT_PY_START 0 #define PY_MONITORING_EVENT_PY_RESUME 1 @@ -29,7 +30,7 @@ extern "C" { #define PY_MONITORING_INSTRUMENTED_EVENTS 10 -/* Exceptional events */ +/* Other events, mainly exceptions */ #define PY_MONITORING_EVENT_RAISE 10 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 @@ -37,7 +38,7 @@ extern "C" { #define PY_MONITORING_EVENT_PY_THROW 13 -/* Grouped events */ +/* Ancilliary events */ #define PY_MONITORING_EVENT_C_RETURN 14 #define PY_MONITORING_EVENT_C_RAISE 15 @@ -45,10 +46,9 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; -/* Reserved IDs */ - -#define PY_INSTRUMENT_SYS_PROFILE 6 -#define PY_INSTRUMENT_SYS_TRACE 7 +/* Tool IDs */ +#define PY_MONITORING_SYS_PROFILE_ID 6 +#define PY_MONITORING_SYS_TRACE_ID 7 PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 39f5add0019989..322c8c82d0a89c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -145,7 +145,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -3187,7 +3187,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/ceval.c b/Python/ceval.c index c2b0dd5ea95776..19315c87c62f1b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -617,7 +617,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_FetchStackPointer(frame); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (err) { \ next_instr = (dest)+1; \ goto error; \ @@ -723,7 +723,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc108f01a2062d..e37d122986a27b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,7 +21,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -4421,7 +4421,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 305cd178c81bc5..ef729362b9b1ee 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1600,7 +1600,7 @@ module monitoring static int check_valid_tool(int tool_id) { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) { PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id); return -1; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index a0a47fa6e6abea..677910e9de8c8e 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -388,32 +388,32 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func2, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_CALL, PY_MONITORING_EVENT_CALL, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_RETURN, PY_MONITORING_EVENT_C_RETURN, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_EXCEPTION, PY_MONITORING_EVENT_C_RAISE, -1)) { return -1; @@ -433,10 +433,10 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND); - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, 0); } return 0; } @@ -459,52 +459,52 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_return, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_yield, PyTrace_RETURN, PY_MONITORING_EVENT_PY_YIELD, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, PY_MONITORING_EVENT_RAISE, PY_MONITORING_EVENT_STOP_ITERATION)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_line_func, PyTrace_LINE, PY_MONITORING_EVENT_LINE, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_instruction_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { return -1; @@ -531,10 +531,10 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, 0); } return 0; From 2d9a3803b4dbff955e68a037169d98c79c910478 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:22:01 +0100 Subject: [PATCH 099/116] Add various IDs as defined in the PEP. --- Include/internal/pycore_instruments.h | 8 ++++++++ Python/instrumentation.c | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 9615efcaf0a35b..fa5eedcefb8f4f 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -47,6 +47,14 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ + +/* These are defined in PEP 669 for convenience to avoid clashes */ +#define PY_MONITORING_DEBUGGER_ID 0 +#define PY_MONITORING_COVERAGE_ID 1 +#define PY_MONITORING_PROFILER_ID 2 +#define PY_MONITORING_OPTIMIZER_ID 5 + +/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ef729362b9b1ee..016c5c47e5a02e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1979,6 +1979,22 @@ PyObject *_Py_CreateMonitoringObject(void) goto error; } } + PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + err = PyObject_SetAttrString(mod, "PROFILER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); + Py_DECREF(val); + if (err) goto error; return mod; error: Py_DECREF(mod); From 29f41e7a3263f284474ae31cb38254d55540290a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:25:21 +0100 Subject: [PATCH 100/116] Remove _PyThreadState_UpdateTracingState. --- Include/internal/pycore_pystate.h | 6 ------ Python/ceval.c | 1 - 2 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index f9b5077f51fe1f..6e5f2289cb6b95 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -133,12 +133,6 @@ extern void _PyThreadState_BindDetached(PyThreadState *); extern void _PyThreadState_UnbindDetached(PyThreadState *); -static inline void -_PyThreadState_UpdateTracingState(PyThreadState *tstate) -{ -} - - /* Other */ PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( diff --git a/Python/ceval.c b/Python/ceval.c index 19315c87c62f1b..72f1b47e3a303d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1967,7 +1967,6 @@ PyThreadState_LeaveTracing(PyThreadState *tstate) { assert(tstate->tracing > 0); tstate->tracing--; - _PyThreadState_UpdateTracingState(tstate); } From 38b7f4350a7695cfc2e5e1a7b6b4c327fe891f0c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:48:24 +0100 Subject: [PATCH 101/116] Formatting fixes, and check error codes. --- Include/internal/pycore_instruments.h | 2 +- Objects/frameobject.c | 4 +-- Python/ceval.c | 31 +++++++++---------- Python/instrumentation.c | 44 +++++++++++++-------------- 4 files changed, 38 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index fa5eedcefb8f4f..e94d8755546efd 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -61,7 +61,7 @@ typedef uint32_t _PyMonitoringEventSet; PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); -void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); extern int _Py_call_instrumentation(PyThreadState *tstate, int event, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 86cf579710956d..2facffaca66c0b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -668,13 +668,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore "f_lineno can only be set in a trace function"); return -1; } - switch(what_event) { + switch (what_event) { case PY_MONITORING_EVENT_PY_RESUME: case PY_MONITORING_EVENT_JUMP: case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: - /* OK */ + /* Setting f_lineno is allowed for the above events */ break; case PY_MONITORING_EVENT_PY_START: PyErr_Format(PyExc_ValueError, diff --git a/Python/ceval.c b/Python/ceval.c index 72f1b47e3a303d..8d2fc2c4d0e9e8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1865,15 +1865,13 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, } static int -do_monitor_exc( - PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event -) { +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ assert(event < PY_MONITORING_UNGROUPED_EVENTS); PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); - int err; - err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); if (err == 0) { PyErr_SetRaisedException(exc); } @@ -1901,8 +1899,8 @@ no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) } static void -monitor_raise( - PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { return; @@ -1911,8 +1909,8 @@ monitor_raise( } static int -monitor_stop_iteration( - PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; @@ -1922,10 +1920,9 @@ monitor_stop_iteration( static void monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } @@ -1935,8 +1932,8 @@ monitor_unwind(PyThreadState *tstate, static void monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return; @@ -1946,8 +1943,8 @@ monitor_handled(PyThreadState *tstate, static void monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 016c5c47e5a02e..6588eff344cb8f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -265,17 +265,6 @@ instruction_length(PyCodeObject *code, int offset) return 1 + _PyOpcode_Caches[opcode]; } -typedef struct _Instruction { - uint8_t extended_args; - uint8_t deinstrumented_opcode; - uint8_t actual_opcode; - uint8_t length; - /* Whether a prefix instrument like INSTRUMENTED_INSTRUCTION is attached */ - bool is_prefix_instrumented; - /* Whether a specialized instruction like INSTRUMENTED_RETURN_VALUE is used */ - bool is_specialized_instrumented; -} Instruction; - #ifdef INSTRUMENT_DEBUG static void @@ -628,7 +617,7 @@ instrument(PyCodeObject *code, int i) } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode_ptr = &lines->original_opcode; + opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); @@ -1099,7 +1088,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1152,7 +1141,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1265,7 +1254,7 @@ initialize_lines(PyCodeObject *code) case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND - * RESUME must not be instrumented with INSTRUMENT_LINE */ + * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; break; default: @@ -1519,7 +1508,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) -static void +static int instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); @@ -1529,7 +1518,9 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyInterpreterFrame *frame = ts->cframe->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { - _Py_Instrument(frame->f_code, interp); + if (_Py_Instrument(frame->f_code, interp)) { + return -1; + } } frame = frame->previous; } @@ -1537,6 +1528,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { ts = PyThreadState_Next(ts); HEAD_UNLOCK(runtime); } + return 0; } static void @@ -1551,7 +1543,7 @@ set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) } } -void +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); @@ -1559,11 +1551,11 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - return; + return 0; } set_events(&interp->monitors, tool_id, events); interp->monitoring_version++; - instrument_all_executing_code_objects(interp); + return instrument_all_executing_code_objects(interp); } int @@ -1585,7 +1577,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version = UINT64_MAX; } - _Py_Instrument(code, interp); + if (_Py_Instrument(code, interp)) { + return -1; + } return 0; } @@ -1768,7 +1762,9 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) return NULL; } event_set &= ~C_RETURN_EVENTS; - _PyMonitoring_SetEvents(tool_id, event_set); + if (_PyMonitoring_SetEvents(tool_id, event_set)) { + return NULL; + } Py_RETURN_NONE; } @@ -1862,7 +1858,9 @@ monitoring_restart_events_impl(PyObject *module) PyInterpreterState *interp = _PyInterpreterState_Get(); interp->last_restart_version = interp->monitoring_version + 1; interp->monitoring_version = interp->last_restart_version + 1; - instrument_all_executing_code_objects(interp); + if (instrument_all_executing_code_objects(interp)) { + return NULL; + } Py_RETURN_NONE; } From 9a40dad178c067fa6a977943823107b4d9c0a0cc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:03:24 +0100 Subject: [PATCH 102/116] Make sure tool API function names match PEP. --- Lib/test/test_monitoring.py | 8 ++++---- Python/clinic/instrumentation.c.h | 32 +++++++++++++++---------------- Python/instrumentation.c | 20 ++++++++----------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 8cc2fa8792f4fa..162466cf78c091 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -38,8 +38,8 @@ class MonitoringBaseTest(unittest.TestCase): def test_has_objects(self): m = sys.monitoring m.events - m.use_tool - m.free_tool + m.use_tool_id + m.free_tool_id m.get_tool m.get_events m.set_events @@ -51,9 +51,9 @@ def test_has_objects(self): m.MISSING def test_tool(self): - sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") - sys.monitoring.free_tool(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) sys.monitoring.set_events(TEST_TOOL, 15) self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 57021191e29df6..deaef2e21902da 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -8,25 +8,25 @@ preserve #endif -PyDoc_STRVAR(monitoring_use_tool__doc__, -"use_tool($module, tool_id, name, /)\n" +PyDoc_STRVAR(monitoring_use_tool_id__doc__, +"use_tool_id($module, tool_id, name, /)\n" "--\n" "\n"); -#define MONITORING_USE_TOOL_METHODDEF \ - {"use_tool", _PyCFunction_CAST(monitoring_use_tool), METH_FASTCALL, monitoring_use_tool__doc__}, +#define MONITORING_USE_TOOL_ID_METHODDEF \ + {"use_tool_id", _PyCFunction_CAST(monitoring_use_tool_id), METH_FASTCALL, monitoring_use_tool_id__doc__}, static PyObject * -monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name); +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name); static PyObject * -monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int tool_id; PyObject *name; - if (!_PyArg_CheckPositional("use_tool", nargs, 2, 2)) { + if (!_PyArg_CheckPositional("use_tool_id", nargs, 2, 2)) { goto exit; } tool_id = _PyLong_AsInt(args[0]); @@ -34,25 +34,25 @@ monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } name = args[1]; - return_value = monitoring_use_tool_impl(module, tool_id, name); + return_value = monitoring_use_tool_id_impl(module, tool_id, name); exit: return return_value; } -PyDoc_STRVAR(monitoring_free_tool__doc__, -"free_tool($module, tool_id, /)\n" +PyDoc_STRVAR(monitoring_free_tool_id__doc__, +"free_tool_id($module, tool_id, /)\n" "--\n" "\n"); -#define MONITORING_FREE_TOOL_METHODDEF \ - {"free_tool", (PyCFunction)monitoring_free_tool, METH_O, monitoring_free_tool__doc__}, +#define MONITORING_FREE_TOOL_ID_METHODDEF \ + {"free_tool_id", (PyCFunction)monitoring_free_tool_id, METH_O, monitoring_free_tool_id__doc__}, static PyObject * -monitoring_free_tool_impl(PyObject *module, int tool_id); +monitoring_free_tool_id_impl(PyObject *module, int tool_id); static PyObject * -monitoring_free_tool(PyObject *module, PyObject *arg) +monitoring_free_tool_id(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int tool_id; @@ -61,7 +61,7 @@ monitoring_free_tool(PyObject *module, PyObject *arg) if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_free_tool_impl(module, tool_id); + return_value = monitoring_free_tool_id_impl(module, tool_id); exit: return return_value; @@ -298,4 +298,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=9e5f270f3ce1e945 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5d03f9aab1fa692 input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6588eff344cb8f..e8695637e56606 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1602,7 +1602,7 @@ check_valid_tool(int tool_id) } /*[clinic input] -monitoring.use_tool +monitoring.use_tool_id tool_id: int name: object @@ -1611,10 +1611,8 @@ monitoring.use_tool [clinic start generated code]*/ static PyObject * -monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) -/*[clinic end generated code: output=d00b74d147bab1e3 input=506e604e1ea75567]*/ - -/*[clinic end generated code]*/ +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) +/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/ { if (check_valid_tool(tool_id)) { return NULL; @@ -1633,7 +1631,7 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) } /*[clinic input] -monitoring.free_tool +monitoring.free_tool_id tool_id: int / @@ -1641,10 +1639,8 @@ monitoring.free_tool [clinic start generated code]*/ static PyObject * -monitoring_free_tool_impl(PyObject *module, int tool_id) -/*[clinic end generated code: output=7893bfdad26f51fa input=919fecb6f63130ed]*/ - -/*[clinic end generated code]*/ +monitoring_free_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/ { if (check_valid_tool(tool_id)) { return NULL; @@ -1926,8 +1922,8 @@ monitoring__all_events_impl(PyObject *module) } static PyMethodDef methods[] = { - MONITORING_USE_TOOL_METHODDEF - MONITORING_FREE_TOOL_METHODDEF + MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_FREE_TOOL_ID_METHODDEF MONITORING_GET_TOOL_METHODDEF MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF From a551d65f4dcd1eb5877a203e36aa058967951e8c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:41:05 +0100 Subject: [PATCH 103/116] A bit more cleanup. --- .../2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst | 2 +- Python/instrumentation.c | 11 +++-------- Python/pystate.c | 5 +++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst index 4fd6fa29bc3648..631ef4c7890450 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst @@ -1 +1 @@ -Adds low overhead monitoring support. See PEP 669 for full details. +Implement :pep:`669` Low Impact Monitoring for CPython. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e8695637e56606..7d4633da5149dc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1937,14 +1937,9 @@ static PyMethodDef methods[] = { static struct PyModuleDef monitoring_module = { PyModuleDef_HEAD_INIT, - "sys.monitoring", - NULL, - -1, /* multiple "initialization" just copies the module dict. */ - methods, - NULL, - NULL, - NULL, - NULL + .m_name = "sys.monitoring", + .m_size = -1, /* multiple "initialization" just copies the module dict. */ + .m_methods = methods, }; PyObject *_Py_CreateMonitoringObject(void) diff --git a/Python/pystate.c b/Python/pystate.c index 5b2dc5a97a79a8..3818434c485194 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -671,7 +671,7 @@ init_interpreter(PyInterpreterState *interp, } } interp->sys_profile_initialized = false; - interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -818,7 +818,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } } interp->sys_profile_initialized = false; - interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } @@ -880,6 +880,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; + interp->f_opcode_trace_set = false; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From 495130672b7aa49aac10e6af9bf8dcc9f5af3d5a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:44:17 +0100 Subject: [PATCH 104/116] Tidy up imports. --- Lib/test/test_monitoring.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 162466cf78c091..685e6e8f16d310 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1,12 +1,12 @@ """Test suite for the sys.monitoring.""" -import sys -import unittest -import enum -import operator +import collections import functools +import operator +import sys import types -import collections +import unittest + PAIR = (0,1) From 44a031e255678acbde5e7801443b9366081bfe94 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:53:35 +0100 Subject: [PATCH 105/116] Use clinic return converters. --- Python/clinic/instrumentation.c.h | 20 +++++++++++++++----- Python/instrumentation.c | 22 +++++++++++----------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index deaef2e21902da..98ec460071f399 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -140,7 +140,7 @@ PyDoc_STRVAR(monitoring_get_events__doc__, #define MONITORING_GET_EVENTS_METHODDEF \ {"get_events", (PyCFunction)monitoring_get_events, METH_O, monitoring_get_events__doc__}, -static PyObject * +static int monitoring_get_events_impl(PyObject *module, int tool_id); static PyObject * @@ -148,12 +148,17 @@ monitoring_get_events(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int tool_id; + int _return_value; tool_id = _PyLong_AsInt(arg); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_get_events_impl(module, tool_id); + _return_value = monitoring_get_events_impl(module, tool_id); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -202,7 +207,7 @@ PyDoc_STRVAR(monitoring_get_local_events__doc__, #define MONITORING_GET_LOCAL_EVENTS_METHODDEF \ {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, -static PyObject * +static int monitoring_get_local_events_impl(PyObject *module, PyObject *code, int tool_id); @@ -212,6 +217,7 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *return_value = NULL; PyObject *code; int tool_id; + int _return_value; if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { goto exit; @@ -221,7 +227,11 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_get_local_events_impl(module, code, tool_id); + _return_value = monitoring_get_local_events_impl(module, code, tool_id); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -298,4 +308,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=f5d03f9aab1fa692 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aa896325abcb6ca9 input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 7d4633da5149dc..97215f8167bde4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1714,23 +1714,23 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } /*[clinic input] -monitoring.get_events +monitoring.get_events -> int tool_id: int / [clinic start generated code]*/ -static PyObject * +static int monitoring_get_events_impl(PyObject *module, int tool_id) -/*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ +/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/ { if (check_valid_tool(tool_id)) { - return NULL; + return -1; } _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); - return PyLong_FromUnsignedLong(event_set); + return event_set; } /*[clinic input] @@ -1765,7 +1765,7 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) } /*[clinic input] -monitoring.get_local_events +monitoring.get_local_events -> int code: object tool_id: int @@ -1773,20 +1773,20 @@ monitoring.get_local_events [clinic start generated code]*/ -static PyObject * +static int monitoring_get_local_events_impl(PyObject *module, PyObject *code, int tool_id) -/*[clinic end generated code: output=22fad50d2e6404a7 input=c14650d27de48264]*/ +/*[clinic end generated code: output=d37536a48b3b2332 input=e33227382525a36d]*/ { if (!PyCode_Check(code)) { PyErr_Format( PyExc_TypeError, "code must be a code object" ); - return NULL; + return -1; } if (check_valid_tool(tool_id)) { - return NULL; + return -1; } _PyMonitoringEventSet event_set = 0; for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { @@ -1795,7 +1795,7 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, event_set |= (1 << e); } } - return PyLong_FromUnsignedLong(event_set); + return event_set; } /*[clinic input] From c2155b70c99047f5c1098432aa25d4081547416f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 23:38:00 +0100 Subject: [PATCH 106/116] Address code review. --- Include/cpython/code.h | 5 ++++- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_interp.h | 1 - Lib/opcode.py | 2 ++ Lib/test/test_monitoring.py | 30 +++++++++++++++++++++++++++--- Python/clinic/instrumentation.c.h | 30 +++++++++++++++--------------- Python/instrumentation.c | 16 ++++++++-------- Python/specialize.c | 2 +- 8 files changed, 58 insertions(+), 30 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index bf7a3b666b2d39..6bead361c79245 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,8 +8,11 @@ extern "C" { #endif -#define PY_MONITORING_EVENTS 16 + +/* Count of all "real" monitoring events (not derived from other events) */ #define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Count of all monitoring events */ +#define PY_MONITORING_EVENTS 16 /* Table of which tools are active for each monitored event. */ typedef struct _Py_Monitors { diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index acdfe17ee7cd99..856297aaea8ab4 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -145,7 +145,7 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1ef416ce19dada..a07510a4b26c46 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -219,7 +219,6 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); - #ifdef __cplusplus } #endif diff --git a/Lib/opcode.py b/Lib/opcode.py index 5aa32d3cb77613..78ae36f34da7e9 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -92,6 +92,8 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_INVERT', 15) +# We reserve 17 as it is the initial value for the specializing counter +# This helps us catch cases where we attempt to execute a cache. def_op('RESERVED', 17) def_op('BINARY_SUBSCR', 25) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 685e6e8f16d310..cdc5ca19516a69 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -940,14 +940,14 @@ def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecor ev = recorder.event_type sys.monitoring.register_callback(tool, ev, recorder(event_list)) all_events |= ev - sys.monitoring.set_local_events(func.__code__, tool, all_events) + sys.monitoring.set_local_events(tool, func.__code__, all_events) func() - sys.monitoring.set_local_events(func.__code__, tool, 0) + sys.monitoring.set_local_events(tool, func.__code__, 0) for recorder in recorders: sys.monitoring.register_callback(tool, recorder.event_type, None) self.assertEqual(event_list, expected) finally: - sys.monitoring.set_local_events(func.__code__, tool, 0) + sys.monitoring.set_local_events(tool, func.__code__, 0) for recorder in recorders: sys.monitoring.register_callback(tool, recorder.event_type, None) @@ -996,3 +996,27 @@ def func3(): ('line', 'func3', 4), ('line', 'func3', 5), ('line', 'func3', 6)]) + + +class TestSetGetEvents(unittest.TestCase, MonitoringTestBase): + + def test_global(self): + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + sys.monitoring.set_events(TEST_TOOL2,0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), 0) + + def test_local(self): + code = f1.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 98ec460071f399..cf3984ca24bbfe 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -200,7 +200,7 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(monitoring_get_local_events__doc__, -"get_local_events($module, code, tool_id, /)\n" +"get_local_events($module, tool_id, code, /)\n" "--\n" "\n"); @@ -208,26 +208,26 @@ PyDoc_STRVAR(monitoring_get_local_events__doc__, {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, static int -monitoring_get_local_events_impl(PyObject *module, PyObject *code, - int tool_id); +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code); static PyObject * monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *code; int tool_id; + PyObject *code; int _return_value; if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { goto exit; } - code = args[0]; - tool_id = _PyLong_AsInt(args[1]); + tool_id = _PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - _return_value = monitoring_get_local_events_impl(module, code, tool_id); + code = args[1]; + _return_value = monitoring_get_local_events_impl(module, tool_id, code); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -238,7 +238,7 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t } PyDoc_STRVAR(monitoring_set_local_events__doc__, -"set_local_events($module, code, tool_id, event_set, /)\n" +"set_local_events($module, tool_id, code, event_set, /)\n" "--\n" "\n"); @@ -246,30 +246,30 @@ PyDoc_STRVAR(monitoring_set_local_events__doc__, {"set_local_events", _PyCFunction_CAST(monitoring_set_local_events), METH_FASTCALL, monitoring_set_local_events__doc__}, static PyObject * -monitoring_set_local_events_impl(PyObject *module, PyObject *code, - int tool_id, int event_set); +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set); static PyObject * monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *code; int tool_id; + PyObject *code; int event_set; if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { goto exit; } - code = args[0]; - tool_id = _PyLong_AsInt(args[1]); + tool_id = _PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } + code = args[1]; event_set = _PyLong_AsInt(args[2]); if (event_set == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_set_local_events_impl(module, code, tool_id, event_set); + return_value = monitoring_set_local_events_impl(module, tool_id, code, event_set); exit: return return_value; @@ -308,4 +308,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=aa896325abcb6ca9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=11cc0803875b3ffa input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 97215f8167bde4..688381049aaa30 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1767,16 +1767,16 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) /*[clinic input] monitoring.get_local_events -> int - code: object tool_id: int + code: object / [clinic start generated code]*/ static int -monitoring_get_local_events_impl(PyObject *module, PyObject *code, - int tool_id) -/*[clinic end generated code: output=d37536a48b3b2332 input=e33227382525a36d]*/ +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code) +/*[clinic end generated code: output=d3e92c1c9c1de8f9 input=bb0f927530386a94]*/ { if (!PyCode_Check(code)) { PyErr_Format( @@ -1801,17 +1801,17 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, /*[clinic input] monitoring.set_local_events - code: object tool_id: int + code: object event_set: int / [clinic start generated code]*/ static PyObject * -monitoring_set_local_events_impl(PyObject *module, PyObject *code, - int tool_id, int event_set) -/*[clinic end generated code: output=65d616f95cbb76d8 input=2706fbfe062404bf]*/ +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set) +/*[clinic end generated code: output=68cc755a65dfea99 input=5655ecd78d937a29]*/ { if (!PyCode_Check(code)) { PyErr_Format( diff --git a/Python/specialize.c b/Python/specialize.c index 74fa7a4f9bc24a..7853d3c94424e9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -274,7 +274,7 @@ _PyCode_Quicken(PyCodeObject *code) for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; opcode = _Py_GetBaseOpcode(code, i); - assert(opcode < 230); + assert(opcode < MIN_INSTRUMENTED_OPCODE); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From b218428a3e41e75eb99d5884f3b061fb74fd90eb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 00:24:20 +0100 Subject: [PATCH 107/116] Address further review comments. --- Python/legacy_tracing.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 677910e9de8c8e..09ee5cf1840c28 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -14,17 +14,11 @@ typedef struct _PyLegacyEventHandler { int event; } _PyLegacyEventHandler; -static void -dealloc(_PyLegacyEventHandler *self) -{ - PyObject_Free(self); -} - - /* The Py_tracefunc function expects the following arguments: - * frame: FrameObject - * kind: c-int - * arg: The arg + * obj: the trace object (PyObject *) + * frame: the current frame (PyFrameObject *) + * kind: the kind of event, see PyTrace_XXX #defines (int) + * arg: The arg (a PyObject *) */ static PyObject * @@ -34,7 +28,7 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); @@ -101,8 +95,6 @@ sys_profile_call_or_return( Py_RETURN_NONE; } - - static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -110,7 +102,7 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -139,7 +131,7 @@ sys_trace_exception_func( if (tb == NULL) { tb = Py_NewRef(Py_None); } - PyObject * tuple = PyTuple_Pack(3, type, exc, tb); + PyObject *tuple = PyTuple_Pack(3, type, exc, tb); Py_DECREF(tb); if (tuple == NULL) { return NULL; @@ -190,7 +182,7 @@ sys_trace_instruction_func( ) { assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -213,7 +205,7 @@ sys_trace_instruction_func( static PyObject * trace_line( PyThreadState *tstate, _PyLegacyEventHandler *self, - PyFrameObject* frame, int line + PyFrameObject *frame, int line ) { if (!frame->f_trace_lines) { Py_RETURN_NONE; @@ -246,7 +238,7 @@ sys_trace_line_func( assert(PyVectorcall_NARGS(nargsf) == 2); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -276,7 +268,7 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -314,10 +306,11 @@ sys_trace_exception_handled( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 3); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); PyCodeObject *code = (PyCodeObject *)args[0]; assert(PyCode_Check(code)); assert(code == frame->f_frame->f_code); + assert(PyLong_Check(args[1])); int offset = _PyLong_AsInt(args[1]); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ @@ -334,9 +327,10 @@ PyTypeObject _PyLegacyEventHandler_Type = { _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), - .tp_dealloc = (destructor)dealloc, + .tp_dealloc = (destructor)PyObject_Free, .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_call = PyVectorcall_Call, }; From 9c0429e6ab32f0554341b9ab56700a74e609d9a6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 04:55:55 +0100 Subject: [PATCH 108/116] Use byte offset, to conform to the convention used in other APIs taking code offsets. --- Lib/test/test_monitoring.py | 72 ++++++++++++++++++------------------- Python/instrumentation.c | 18 ++++++---- Python/legacy_tracing.c | 6 ++-- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index cdc5ca19516a69..c889854b7250be 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -794,16 +794,16 @@ def func1(): self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func1', 1), - ('line', 'func1', 1), ('instruction', 'func1', 2), - ('instruction', 'func1', 3), - ('line', 'func1', 2), + ('line', 'func1', 1), ('instruction', 'func1', 4), - ('instruction', 'func1', 5), - ('line', 'func1', 3), ('instruction', 'func1', 6), - ('instruction', 'func1', 7), + ('line', 'func1', 2), + ('instruction', 'func1', 8), + ('instruction', 'func1', 10), + ('line', 'func1', 3), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), ('line', 'check_events', 11)]) def test_c_call(self): @@ -815,19 +815,19 @@ def func2(): self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func2', 1), - ('line', 'func2', 1), ('instruction', 'func2', 2), - ('instruction', 'func2', 3), - ('line', 'func2', 2), + ('line', 'func2', 1), ('instruction', 'func2', 4), - ('instruction', 'func2', 14), - ('instruction', 'func2', 15), - ('instruction', 'func2', 20), - ('instruction', 'func2', 21), + ('instruction', 'func2', 6), + ('line', 'func2', 2), + ('instruction', 'func2', 8), + ('instruction', 'func2', 28), + ('instruction', 'func2', 30), + ('instruction', 'func2', 40), + ('instruction', 'func2', 42), ('line', 'func2', 3), - ('instruction', 'func2', 22), - ('instruction', 'func2', 23), + ('instruction', 'func2', 44), + ('instruction', 'func2', 46), ('line', 'check_events', 11)]) def test_try_except(self): @@ -842,25 +842,25 @@ def func3(): self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func3', 1), - ('line', 'func3', 1), ('instruction', 'func3', 2), - ('line', 'func3', 2), - ('instruction', 'func3', 3), + ('line', 'func3', 1), ('instruction', 'func3', 4), + ('line', 'func3', 2), + ('instruction', 'func3', 6), + ('instruction', 'func3', 8), ('line', 'func3', 3), - ('instruction', 'func3', 9), - ('instruction', 'func3', 10), - ('instruction', 'func3', 11), + ('instruction', 'func3', 18), + ('instruction', 'func3', 20), + ('instruction', 'func3', 22), ('line', 'func3', 4), - ('instruction', 'func3', 12), + ('instruction', 'func3', 24), ('line', 'func3', 5), - ('instruction', 'func3', 13), - ('instruction', 'func3', 14), - ('instruction', 'func3', 15), + ('instruction', 'func3', 26), + ('instruction', 'func3', 28), + ('instruction', 'func3', 30), ('line', 'func3', 6), - ('instruction', 'func3', 16), - ('instruction', 'func3', 17), + ('instruction', 'func3', 32), + ('instruction', 'func3', 34), ('line', 'check_events', 11)]) class TestInstallIncrementallly(unittest.TestCase): @@ -891,10 +891,10 @@ def func1(): line1 = 1 MUST_INCLUDE_LI = [ - ('instruction', 'func1', 1), - ('line', 'func1', 1), ('instruction', 'func1', 2), - ('instruction', 'func1', 3)] + ('line', 'func1', 1), + ('instruction', 'func1', 4), + ('instruction', 'func1', 6)] def test_line_then_instruction(self): recorders = [ LineRecorder, InstructionRecorder ] @@ -911,11 +911,11 @@ def func2(): len(()) MUST_INCLUDE_CI = [ - ('instruction', 'func2', 1), + ('instruction', 'func2', 2), ('call', 'func2', sys.monitoring.MISSING), ('call', 'len', ()), - ('instruction', 'func2', 6), - ('instruction', 'func2', 7)] + ('instruction', 'func2', 12), + ('instruction', 'func2', 14)] diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 688381049aaa30..436c2b48d178ab 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -924,8 +924,11 @@ call_instrumentation_vector( assert(instrumentation_cross_checks(tstate->interp, code)); assert(args[1] == NULL); args[1] = (PyObject *)code; - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); + int offset = (int)(instr - _PyCode_CODE(code)); + /* Offset visible to user should be the offset in bytes, as that is the + * convention for APIs involving code offsets. */ + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); if (offset_obj == NULL) { return -1; } @@ -996,8 +999,8 @@ _Py_call_instrumentation_jump( assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int to = target - _PyCode_CODE(code); - PyObject *to_obj = PyLong_FromLong(to); + int to = (int)(target - _PyCode_CODE(code)); + PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return -1; } @@ -1066,7 +1069,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); - int i = instr - _PyCode_CODE(code); + int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; @@ -1121,7 +1124,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); + int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; @@ -1134,7 +1137,8 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* (interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] | code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); - PyObject *offset_obj = PyLong_FromSsize_t(offset); + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); if (offset_obj == NULL) { return -1; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 09ee5cf1840c28..6e6deea7b818ce 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -264,9 +264,9 @@ sys_trace_jump_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 3); - int from = _PyLong_AsInt(args[1]); + int from = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); assert(from >= 0); - int to = _PyLong_AsInt(args[2]); + int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); assert(to >= 0); PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { @@ -311,7 +311,7 @@ sys_trace_exception_handled( assert(PyCode_Check(code)); assert(code == frame->f_frame->f_code); assert(PyLong_Check(args[1])); - int offset = _PyLong_AsInt(args[1]); + int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int line = _Py_Instrumentation_GetLine(code, offset); From c6bf38e29d126a6b812693fc89f59736c05363c9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 18:29:16 +0100 Subject: [PATCH 109/116] Address code review. --- Objects/object.c | 2 + Python/bytecodes.c | 40 +- Python/ceval.c | 15 - Python/ceval_macros.h | 15 + Python/generated_cases.c.h | 986 +++++++++++++++++++------------------ Python/opcode_metadata.h | 18 +- 6 files changed, 540 insertions(+), 536 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 9dd5eb998217f6..fcd6fa59af8b65 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1978,6 +1978,7 @@ extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; +extern PyTypeObject _PyLegacyEventHandler_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2075,6 +2076,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyLegacyEventHandler_Type, &_PyInterpreterID_Type, &_PyLineIterator, &_PyManagedBuffer_Type, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 322c8c82d0a89c..bf26d13c5884ca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -132,9 +132,25 @@ dummy_func( inst(NOP, (--)) { } - inst(INSTRUMENTED_RESUME, (--)) { - /* Check monitoring *before* calling instrument */ + inst(RESUME, (--)) { + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + ERROR_IF(err, error); + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + } + + inst(INSTRUMENTED_RESUME, (--)) { + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -158,20 +174,6 @@ dummy_func( } } - inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); - /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(frame->f_code, tstate->interp); - ERROR_IF(err, error); - next_instr--; - } - else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } - } - inst(LOAD_CLOSURE, (-- value)) { /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); @@ -2159,7 +2161,7 @@ dummy_func( // Common case: no jump, leave it to the code generator } - inst(INSTRUMENTED_FOR_ITER) { + inst(INSTRUMENTED_FOR_ITER, ( -- )) { _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2999,7 +3001,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(INSTRUMENTED_CALL_FUNCTION_EX) { + inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } @@ -3218,12 +3220,10 @@ dummy_func( DISPATCH_GOTO(); } - // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); } - // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); diff --git a/Python/ceval.c b/Python/ceval.c index 8d2fc2c4d0e9e8..a38c9ec9ad5f9c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -610,21 +610,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } -// If a trace function sets a new f_lineno and -// *then* raises, we use the destination when searching -// for an exception handler, displaying the traceback, and so on -#define INSTRUMENTED_JUMP(src, dest, event) \ -do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ - next_instr = (dest)+1; \ - goto error; \ - } \ - next_instr = frame->prev_instr; \ -} while (0); - /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 2fe740991497b0..485771ac65a767 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -327,3 +327,18 @@ do { \ _Py_DECREF_NO_DEALLOC(right); \ } \ } while (0) + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (err) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ + next_instr = frame->prev_instr; \ +} while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e37d122986a27b..a2877cb86b63b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,10 +7,29 @@ DISPATCH(); } - TARGET(INSTRUMENTED_RESUME) { + TARGET(RESUME) { #line 136 "Python/bytecodes.c" - /* Check monitoring *before* calling instrument */ + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + if (err) goto error; + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + #line 24 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + #line 150 "Python/bytecodes.c" + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -32,35 +51,18 @@ goto handle_eval_breaker; } } - #line 36 "Python/generated_cases.c.h" - DISPATCH(); - } - - TARGET(RESUME) { - #line 162 "Python/bytecodes.c" - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); - /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(frame->f_code, tstate->interp); - if (err) goto error; - next_instr--; - } - else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } - #line 53 "Python/generated_cases.c.h" + #line 55 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 176 "Python/bytecodes.c" + #line 178 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 64 "Python/generated_cases.c.h" + #line 66 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -68,11 +70,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 183 "Python/bytecodes.c" + #line 185 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 76 "Python/generated_cases.c.h" + #line 78 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -80,11 +82,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 88 "Python/generated_cases.c.h" + #line 90 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -93,10 +95,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 100 "Python/generated_cases.c.h" + #line 102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -104,9 +106,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 110 "Python/generated_cases.c.h" + #line 112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -116,21 +118,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 124 "Python/generated_cases.c.h" + #line 126 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 134 "Python/generated_cases.c.h" + #line 136 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -144,20 +146,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 152 "Python/generated_cases.c.h" + #line 154 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 161 "Python/generated_cases.c.h" + #line 163 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -170,18 +172,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 176 "Python/generated_cases.c.h" + #line 178 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 185 "Python/generated_cases.c.h" + #line 187 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -193,16 +195,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 199 "Python/generated_cases.c.h" + #line 201 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 206 "Python/generated_cases.c.h" + #line 208 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -213,20 +215,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 220 "Python/generated_cases.c.h" + #line 222 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 230 "Python/generated_cases.c.h" + #line 232 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -237,8 +239,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 210 "Python/bytecodes.c" - #line 242 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 244 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -246,9 +248,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 214 "Python/bytecodes.c" + #line 216 "Python/bytecodes.c" res = NULL; - #line 252 "Python/generated_cases.c.h" + #line 254 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -259,14 +261,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 210 "Python/bytecodes.c" - #line 264 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 266 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 210 "Python/bytecodes.c" - #line 270 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 272 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -276,7 +278,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 220 "Python/bytecodes.c" + #line 222 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -286,7 +288,7 @@ } PyErr_SetRaisedException(NULL); } - #line 290 "Python/generated_cases.c.h" + #line 292 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -296,9 +298,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 233 "Python/bytecodes.c" + #line 235 "Python/bytecodes.c" Py_DECREF(receiver); - #line 302 "Python/generated_cases.c.h" + #line 304 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -307,7 +309,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 237 "Python/bytecodes.c" + #line 239 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -316,7 +318,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 320 "Python/generated_cases.c.h" + #line 322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -325,13 +327,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 248 "Python/bytecodes.c" + #line 250 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 331 "Python/generated_cases.c.h" + #line 333 "Python/generated_cases.c.h" Py_DECREF(value); - #line 250 "Python/bytecodes.c" + #line 252 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 335 "Python/generated_cases.c.h" + #line 337 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -339,11 +341,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 254 "Python/bytecodes.c" + #line 256 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 345 "Python/generated_cases.c.h" + #line 347 "Python/generated_cases.c.h" Py_DECREF(value); - #line 256 "Python/bytecodes.c" + #line 258 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -352,7 +354,7 @@ res = Py_False; } Py_INCREF(res); - #line 356 "Python/generated_cases.c.h" + #line 358 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -360,13 +362,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 267 "Python/bytecodes.c" + #line 269 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 366 "Python/generated_cases.c.h" + #line 368 "Python/generated_cases.c.h" Py_DECREF(value); - #line 269 "Python/bytecodes.c" + #line 271 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 370 "Python/generated_cases.c.h" + #line 372 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -375,7 +377,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 286 "Python/bytecodes.c" + #line 288 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -383,7 +385,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 387 "Python/generated_cases.c.h" + #line 389 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -394,14 +396,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 296 "Python/bytecodes.c" + #line 298 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 405 "Python/generated_cases.c.h" + #line 407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -412,7 +414,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 305 "Python/bytecodes.c" + #line 307 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -420,7 +422,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 424 "Python/generated_cases.c.h" + #line 426 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -431,13 +433,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 315 "Python/bytecodes.c" + #line 317 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 441 "Python/generated_cases.c.h" + #line 443 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -448,7 +450,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 323 "Python/bytecodes.c" + #line 325 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -456,7 +458,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 460 "Python/generated_cases.c.h" + #line 462 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -466,7 +468,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 339 "Python/bytecodes.c" + #line 341 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -493,7 +495,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 497 "Python/generated_cases.c.h" + #line 499 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -502,14 +504,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 368 "Python/bytecodes.c" + #line 370 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 513 "Python/generated_cases.c.h" + #line 515 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -520,7 +522,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 377 "Python/bytecodes.c" + #line 379 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -528,7 +530,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 532 "Python/generated_cases.c.h" + #line 534 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -541,7 +543,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 395 "Python/bytecodes.c" + #line 397 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -553,12 +555,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 557 "Python/generated_cases.c.h" + #line 559 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 407 "Python/bytecodes.c" + #line 409 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 562 "Python/generated_cases.c.h" + #line 564 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -570,7 +572,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 411 "Python/bytecodes.c" + #line 413 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -583,7 +585,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 587 "Python/generated_cases.c.h" + #line 589 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -594,7 +596,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 426 "Python/bytecodes.c" + #line 428 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -607,7 +609,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 611 "Python/generated_cases.c.h" + #line 613 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -616,7 +618,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 441 "Python/bytecodes.c" + #line 443 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -630,7 +632,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 634 "Python/generated_cases.c.h" + #line 636 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -641,7 +643,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 457 "Python/bytecodes.c" + #line 459 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -655,7 +657,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 659 "Python/generated_cases.c.h" + #line 661 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -666,7 +668,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 473 "Python/bytecodes.c" + #line 475 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -674,14 +676,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 678 "Python/generated_cases.c.h" + #line 680 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 481 "Python/bytecodes.c" + #line 483 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 685 "Python/generated_cases.c.h" + #line 687 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -693,7 +695,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 488 "Python/bytecodes.c" + #line 490 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -714,15 +716,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 718 "Python/generated_cases.c.h" + #line 720 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 511 "Python/bytecodes.c" + #line 513 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 726 "Python/generated_cases.c.h" + #line 728 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -731,13 +733,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 516 "Python/bytecodes.c" + #line 518 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 737 "Python/generated_cases.c.h" + #line 739 "Python/generated_cases.c.h" Py_DECREF(v); - #line 518 "Python/bytecodes.c" + #line 520 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" + #line 743 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -750,7 +752,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 529 "Python/bytecodes.c" + #line 531 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -765,13 +767,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 769 "Python/generated_cases.c.h" + #line 771 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 544 "Python/bytecodes.c" + #line 546 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 775 "Python/generated_cases.c.h" + #line 777 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -781,7 +783,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 548 "Python/bytecodes.c" + #line 550 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -798,7 +800,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 802 "Python/generated_cases.c.h" + #line 804 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -808,13 +810,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 567 "Python/bytecodes.c" + #line 569 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 818 "Python/generated_cases.c.h" + #line 820 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -823,15 +825,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 575 "Python/bytecodes.c" + #line 577 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 830 "Python/generated_cases.c.h" + #line 832 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 578 "Python/bytecodes.c" + #line 580 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 835 "Python/generated_cases.c.h" + #line 837 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -839,14 +841,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 582 "Python/bytecodes.c" + #line 584 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 846 "Python/generated_cases.c.h" + #line 848 "Python/generated_cases.c.h" Py_DECREF(value); - #line 585 "Python/bytecodes.c" + #line 587 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 850 "Python/generated_cases.c.h" + #line 852 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -855,15 +857,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 589 "Python/bytecodes.c" + #line 591 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 862 "Python/generated_cases.c.h" + #line 864 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 592 "Python/bytecodes.c" + #line 594 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 867 "Python/generated_cases.c.h" + #line 869 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -871,7 +873,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 596 "Python/bytecodes.c" + #line 598 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -889,12 +891,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 893 "Python/generated_cases.c.h" + #line 895 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 616 "Python/bytecodes.c" + #line 618 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -905,12 +907,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 909 "Python/generated_cases.c.h" + #line 911 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 629 "Python/bytecodes.c" + #line 631 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -922,12 +924,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 926 "Python/generated_cases.c.h" + #line 928 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 643 "Python/bytecodes.c" + #line 645 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -943,11 +945,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 947 "Python/generated_cases.c.h" + #line 949 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 661 "Python/bytecodes.c" + #line 663 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -960,11 +962,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 964 "Python/generated_cases.c.h" + #line 966 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 676 "Python/bytecodes.c" + #line 678 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -981,13 +983,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 985 "Python/generated_cases.c.h" + #line 987 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 695 "Python/bytecodes.c" + #line 697 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1000,16 +1002,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1004 "Python/generated_cases.c.h" + #line 1006 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 708 "Python/bytecodes.c" + #line 710 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1011 "Python/generated_cases.c.h" + #line 1013 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 713 "Python/bytecodes.c" + #line 715 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1022,7 +1024,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1026 "Python/generated_cases.c.h" + #line 1028 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1030,7 +1032,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 728 "Python/bytecodes.c" + #line 730 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1074,7 +1076,7 @@ } } - #line 1078 "Python/generated_cases.c.h" + #line 1080 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1085,16 +1087,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 775 "Python/bytecodes.c" + #line 777 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1096 "Python/generated_cases.c.h" + #line 1098 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 782 "Python/bytecodes.c" + #line 784 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1112,7 +1114,7 @@ if (iter == NULL) goto pop_1_error; - #line 1116 "Python/generated_cases.c.h" + #line 1118 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1123,7 +1125,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 808 "Python/bytecodes.c" + #line 810 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1169,7 +1171,7 @@ } } Py_DECREF(v); - #line 1173 "Python/generated_cases.c.h" + #line 1175 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1178,7 +1180,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 856 "Python/bytecodes.c" + #line 858 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1193,12 +1195,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1197 "Python/generated_cases.c.h" + #line 1199 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 873 "Python/bytecodes.c" + #line 875 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1216,12 +1218,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1220 "Python/generated_cases.c.h" + #line 1222 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 893 "Python/bytecodes.c" + #line 895 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1238,15 +1240,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1242 "Python/generated_cases.c.h" + #line 1244 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 912 "Python/bytecodes.c" + #line 914 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1250 "Python/generated_cases.c.h" + #line 1252 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1254,7 +1256,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 917 "Python/bytecodes.c" + #line 919 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1272,26 +1274,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1276 "Python/generated_cases.c.h" + #line 1278 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 937 "Python/bytecodes.c" + #line 939 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1285 "Python/generated_cases.c.h" + #line 1287 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 940 "Python/bytecodes.c" + #line 942 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1295 "Python/generated_cases.c.h" + #line 1297 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1302,23 +1304,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 949 "Python/bytecodes.c" + #line 951 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1311 "Python/generated_cases.c.h" + #line 1313 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 954 "Python/bytecodes.c" + #line 956 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1322 "Python/generated_cases.c.h" + #line 1324 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1327,9 +1329,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 963 "Python/bytecodes.c" + #line 965 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1333 "Python/generated_cases.c.h" + #line 1335 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1337,7 +1339,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 967 "Python/bytecodes.c" + #line 969 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1359,7 +1361,7 @@ if (true) goto error; } } - #line 1363 "Python/generated_cases.c.h" + #line 1365 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1367,33 +1369,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 991 "Python/bytecodes.c" + #line 993 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1378 "Python/generated_cases.c.h" + #line 1380 "Python/generated_cases.c.h" Py_DECREF(v); - #line 998 "Python/bytecodes.c" + #line 1000 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1387 "Python/generated_cases.c.h" + #line 1389 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1005 "Python/bytecodes.c" + #line 1007 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1391 "Python/generated_cases.c.h" + #line 1393 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 1009 "Python/bytecodes.c" + #line 1011 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1410,7 +1412,7 @@ name); goto error; } - #line 1414 "Python/generated_cases.c.h" + #line 1416 "Python/generated_cases.c.h" DISPATCH(); } @@ -1418,7 +1420,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 1035 "Python/bytecodes.c" + #line 1037 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1431,11 +1433,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1435 "Python/generated_cases.c.h" + #line 1437 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1048 "Python/bytecodes.c" + #line 1050 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1439 "Python/generated_cases.c.h" + #line 1441 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1445,14 +1447,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1052 "Python/bytecodes.c" + #line 1054 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1456 "Python/generated_cases.c.h" + #line 1458 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1463,7 +1465,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1062 "Python/bytecodes.c" + #line 1064 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1471,7 +1473,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1475 "Python/generated_cases.c.h" + #line 1477 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1482,7 +1484,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1073 "Python/bytecodes.c" + #line 1075 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1490,7 +1492,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1494 "Python/generated_cases.c.h" + #line 1496 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1500,15 +1502,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1084 "Python/bytecodes.c" + #line 1086 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1508 "Python/generated_cases.c.h" + #line 1510 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1088 "Python/bytecodes.c" + #line 1090 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1512 "Python/generated_cases.c.h" + #line 1514 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1519,7 +1521,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1099 "Python/bytecodes.c" + #line 1101 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1535,12 +1537,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1539 "Python/generated_cases.c.h" + #line 1541 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1115 "Python/bytecodes.c" + #line 1117 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1544 "Python/generated_cases.c.h" + #line 1546 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1548,34 +1550,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1119 "Python/bytecodes.c" + #line 1121 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1555 "Python/generated_cases.c.h" + #line 1557 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1122 "Python/bytecodes.c" + #line 1124 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1559 "Python/generated_cases.c.h" + #line 1561 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1126 "Python/bytecodes.c" + #line 1128 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1569 "Python/generated_cases.c.h" + #line 1571 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1129 "Python/bytecodes.c" + #line 1131 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1573 "Python/generated_cases.c.h" + #line 1575 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1133 "Python/bytecodes.c" + #line 1135 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1587,13 +1589,13 @@ } goto error; } - #line 1591 "Python/generated_cases.c.h" + #line 1593 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1147 "Python/bytecodes.c" + #line 1149 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1652,7 +1654,7 @@ } } } - #line 1656 "Python/generated_cases.c.h" + #line 1658 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1663,7 +1665,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1214 "Python/bytecodes.c" + #line 1216 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1715,7 +1717,7 @@ } } null = NULL; - #line 1719 "Python/generated_cases.c.h" + #line 1721 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1729,7 +1731,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1268 "Python/bytecodes.c" + #line 1270 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1740,7 +1742,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1744 "Python/generated_cases.c.h" + #line 1746 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1755,7 +1757,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1281 "Python/bytecodes.c" + #line 1283 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1770,7 +1772,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1774 "Python/generated_cases.c.h" + #line 1776 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1780,16 +1782,16 @@ } TARGET(DELETE_FAST) { - #line 1298 "Python/bytecodes.c" + #line 1300 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1788 "Python/generated_cases.c.h" + #line 1790 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1304 "Python/bytecodes.c" + #line 1306 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1798,12 +1800,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1802 "Python/generated_cases.c.h" + #line 1804 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1315 "Python/bytecodes.c" + #line 1317 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1814,13 +1816,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1818 "Python/generated_cases.c.h" + #line 1820 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1328 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1852,7 +1854,7 @@ } Py_INCREF(value); } - #line 1856 "Python/generated_cases.c.h" + #line 1858 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1860,7 +1862,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1362 "Python/bytecodes.c" + #line 1364 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1868,7 +1870,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1872 "Python/generated_cases.c.h" + #line 1874 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1876,18 +1878,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1372 "Python/bytecodes.c" + #line 1374 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1885 "Python/generated_cases.c.h" + #line 1887 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1379 "Python/bytecodes.c" + #line 1381 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1898,22 +1900,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1902 "Python/generated_cases.c.h" + #line 1904 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1392 "Python/bytecodes.c" + #line 1394 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1911 "Python/generated_cases.c.h" + #line 1913 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1394 "Python/bytecodes.c" + #line 1396 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1917 "Python/generated_cases.c.h" + #line 1919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1923,10 +1925,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1398 "Python/bytecodes.c" + #line 1400 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1930 "Python/generated_cases.c.h" + #line 1932 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1936,10 +1938,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1403 "Python/bytecodes.c" + #line 1405 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1943 "Python/generated_cases.c.h" + #line 1945 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1949,7 +1951,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1408 "Python/bytecodes.c" + #line 1410 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1960,13 +1962,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1964 "Python/generated_cases.c.h" + #line 1966 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1419 "Python/bytecodes.c" + #line 1421 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1970 "Python/generated_cases.c.h" + #line 1972 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1975,13 +1977,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1426 "Python/bytecodes.c" + #line 1428 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1981 "Python/generated_cases.c.h" + #line 1983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1428 "Python/bytecodes.c" + #line 1430 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1985 "Python/generated_cases.c.h" + #line 1987 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1989,7 +1991,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1432 "Python/bytecodes.c" + #line 1434 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2004,7 +2006,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2008 "Python/generated_cases.c.h" + #line 2010 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2014,7 +2016,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1449 "Python/bytecodes.c" + #line 1451 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2022,13 +2024,13 @@ if (map == NULL) goto error; - #line 2026 "Python/generated_cases.c.h" + #line 2028 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1457 "Python/bytecodes.c" + #line 1459 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2032 "Python/generated_cases.c.h" + #line 2034 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2036,7 +2038,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1461 "Python/bytecodes.c" + #line 1463 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2076,7 +2078,7 @@ Py_DECREF(ann_dict); } } - #line 2080 "Python/generated_cases.c.h" + #line 2082 "Python/generated_cases.c.h" DISPATCH(); } @@ -2084,7 +2086,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1503 "Python/bytecodes.c" + #line 1505 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2094,14 +2096,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2098 "Python/generated_cases.c.h" + #line 2100 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1513 "Python/bytecodes.c" + #line 1515 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2105 "Python/generated_cases.c.h" + #line 2107 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2109,7 +2111,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1517 "Python/bytecodes.c" + #line 1519 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2117,12 +2119,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2121 "Python/generated_cases.c.h" + #line 2123 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1525 "Python/bytecodes.c" + #line 1527 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2126 "Python/generated_cases.c.h" + #line 2128 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2130,17 +2132,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1531 "Python/bytecodes.c" + #line 1533 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2139 "Python/generated_cases.c.h" + #line 2141 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1536 "Python/bytecodes.c" + #line 1538 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2144 "Python/generated_cases.c.h" + #line 2146 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2150,13 +2152,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1543 "Python/bytecodes.c" + #line 1545 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2160 "Python/generated_cases.c.h" + #line 2162 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2168,7 +2170,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1566 "Python/bytecodes.c" + #line 1568 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2202,9 +2204,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2206 "Python/generated_cases.c.h" + #line 2208 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1600 "Python/bytecodes.c" + #line 1602 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2213,12 +2215,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2217 "Python/generated_cases.c.h" + #line 2219 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1609 "Python/bytecodes.c" + #line 1611 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2222 "Python/generated_cases.c.h" + #line 2224 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2232,7 +2234,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1614 "Python/bytecodes.c" + #line 1616 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2245,7 +2247,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2249 "Python/generated_cases.c.h" + #line 2251 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2260,7 +2262,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1630 "Python/bytecodes.c" + #line 1632 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2273,7 +2275,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2277 "Python/generated_cases.c.h" + #line 2279 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2288,7 +2290,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1646 "Python/bytecodes.c" + #line 1648 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2315,7 +2317,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2319 "Python/generated_cases.c.h" + #line 2321 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2330,7 +2332,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1676 "Python/bytecodes.c" + #line 1678 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2340,7 +2342,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2344 "Python/generated_cases.c.h" + #line 2346 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2355,7 +2357,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1689 "Python/bytecodes.c" + #line 1691 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2367,7 +2369,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2371 "Python/generated_cases.c.h" + #line 2373 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2381,7 +2383,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1704 "Python/bytecodes.c" + #line 1706 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2404,7 +2406,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2408 "Python/generated_cases.c.h" + #line 2410 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2412,7 +2414,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1729 "Python/bytecodes.c" + #line 1731 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2437,7 +2439,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2441 "Python/generated_cases.c.h" + #line 2443 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2445,7 +2447,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1756 "Python/bytecodes.c" + #line 1758 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2463,7 +2465,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2467 "Python/generated_cases.c.h" + #line 2469 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2474,7 +2476,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1776 "Python/bytecodes.c" + #line 1778 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2513,7 +2515,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2517 "Python/generated_cases.c.h" + #line 2519 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2524,7 +2526,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1817 "Python/bytecodes.c" + #line 1819 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2534,7 +2536,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2538 "Python/generated_cases.c.h" + #line 2540 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2546,7 +2548,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1836 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2559,12 +2561,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2563 "Python/generated_cases.c.h" + #line 2565 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1849 "Python/bytecodes.c" + #line 1851 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2568 "Python/generated_cases.c.h" + #line 2570 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2575,7 +2577,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1853 "Python/bytecodes.c" + #line 1855 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2587,7 +2589,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2591 "Python/generated_cases.c.h" + #line 2593 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2598,7 +2600,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1868 "Python/bytecodes.c" + #line 1870 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2614,7 +2616,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2618 "Python/generated_cases.c.h" + #line 2620 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2625,7 +2627,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1887 "Python/bytecodes.c" + #line 1889 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2638,7 +2640,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2642 "Python/generated_cases.c.h" + #line 2644 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2649,14 +2651,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1902 "Python/bytecodes.c" + #line 1904 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2655 "Python/generated_cases.c.h" + #line 2657 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1904 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2660 "Python/generated_cases.c.h" + #line 2662 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2666,15 +2668,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1908 "Python/bytecodes.c" + #line 1910 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2672 "Python/generated_cases.c.h" + #line 2674 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1910 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2678 "Python/generated_cases.c.h" + #line 2680 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2685,12 +2687,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1915 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2691 "Python/generated_cases.c.h" + #line 2693 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1917 "Python/bytecodes.c" + #line 1919 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2698,10 +2700,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2702 "Python/generated_cases.c.h" + #line 2704 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1925 "Python/bytecodes.c" + #line 1927 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2710,7 +2712,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2714 "Python/generated_cases.c.h" + #line 2716 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2720,21 +2722,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1936 "Python/bytecodes.c" + #line 1938 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2727 "Python/generated_cases.c.h" + #line 2729 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1939 "Python/bytecodes.c" + #line 1941 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2734 "Python/generated_cases.c.h" + #line 2736 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1944 "Python/bytecodes.c" + #line 1946 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2738 "Python/generated_cases.c.h" + #line 2740 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2743,15 +2745,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1948 "Python/bytecodes.c" + #line 1950 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2750 "Python/generated_cases.c.h" + #line 2752 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1951 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2755 "Python/generated_cases.c.h" + #line 2757 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2760,29 +2762,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1955 "Python/bytecodes.c" + #line 1957 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2768 "Python/generated_cases.c.h" + #line 2770 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1961 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" JUMPBY(oparg); - #line 2777 "Python/generated_cases.c.h" + #line 2779 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1965 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2786 "Python/generated_cases.c.h" + #line 2788 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2790,7 +2792,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1971 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2800,9 +2802,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2804 "Python/generated_cases.c.h" + #line 2806 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1981 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2810,14 +2812,14 @@ if (err < 0) goto pop_1_error; } } - #line 2814 "Python/generated_cases.c.h" + #line 2816 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1991 "Python/bytecodes.c" + #line 1993 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2827,9 +2829,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2831 "Python/generated_cases.c.h" + #line 2833 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2001 "Python/bytecodes.c" + #line 2003 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2837,67 +2839,67 @@ if (err < 0) goto pop_1_error; } } - #line 2841 "Python/generated_cases.c.h" + #line 2843 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2011 "Python/bytecodes.c" + #line 2013 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2850 "Python/generated_cases.c.h" + #line 2852 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2013 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2858 "Python/generated_cases.c.h" + #line 2860 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2021 "Python/bytecodes.c" + #line 2023 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2871 "Python/generated_cases.c.h" + #line 2873 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2027 "Python/bytecodes.c" + #line 2029 "Python/bytecodes.c" } - #line 2875 "Python/generated_cases.c.h" + #line 2877 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2031 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2888 "Python/generated_cases.c.h" + #line 2890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2040 "Python/bytecodes.c" + #line 2042 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2901 "Python/generated_cases.c.h" + #line 2903 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2908,16 +2910,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2048 "Python/bytecodes.c" + #line 2050 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2917 "Python/generated_cases.c.h" + #line 2919 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2053 "Python/bytecodes.c" + #line 2055 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2925,7 +2927,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2929 "Python/generated_cases.c.h" + #line 2931 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2934,10 +2936,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2063 "Python/bytecodes.c" + #line 2065 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2941 "Python/generated_cases.c.h" + #line 2943 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2947,10 +2949,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2069 "Python/bytecodes.c" + #line 2071 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2954 "Python/generated_cases.c.h" + #line 2956 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2961,11 +2963,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2075 "Python/bytecodes.c" + #line 2077 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2969 "Python/generated_cases.c.h" + #line 2971 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2974,14 +2976,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2081 "Python/bytecodes.c" + #line 2083 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2981 "Python/generated_cases.c.h" + #line 2983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2084 "Python/bytecodes.c" + #line 2086 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2985 "Python/generated_cases.c.h" + #line 2987 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2989,7 +2991,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2088 "Python/bytecodes.c" + #line 2090 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3012,11 +3014,11 @@ if (iter == NULL) { goto error; } - #line 3016 "Python/generated_cases.c.h" + #line 3018 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2111 "Python/bytecodes.c" + #line 2113 "Python/bytecodes.c" } - #line 3020 "Python/generated_cases.c.h" + #line 3022 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3027,7 +3029,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2130 "Python/bytecodes.c" + #line 2132 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3058,7 +3060,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3062 "Python/generated_cases.c.h" + #line 3064 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3066,7 +3068,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2163 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3092,14 +3094,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3096 "Python/generated_cases.c.h" + #line 3098 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2191 "Python/bytecodes.c" + #line 2193 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3119,7 +3121,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3123 "Python/generated_cases.c.h" + #line 3125 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3129,7 +3131,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2213 "Python/bytecodes.c" + #line 2215 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3149,7 +3151,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3153 "Python/generated_cases.c.h" + #line 3155 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3159,7 +3161,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2235 "Python/bytecodes.c" + #line 2237 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3177,7 +3179,7 @@ if (next == NULL) { goto error; } - #line 3181 "Python/generated_cases.c.h" + #line 3183 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3186,7 +3188,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2255 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3201,14 +3203,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3205 "Python/generated_cases.c.h" + #line 3207 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2272 "Python/bytecodes.c" + #line 2274 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3231,16 +3233,16 @@ Py_DECREF(enter); goto error; } - #line 3235 "Python/generated_cases.c.h" + #line 3237 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2295 "Python/bytecodes.c" + #line 2297 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3244 "Python/generated_cases.c.h" + #line 3246 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3252,7 +3254,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2305 "Python/bytecodes.c" + #line 2307 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3278,16 +3280,16 @@ Py_DECREF(enter); goto error; } - #line 3282 "Python/generated_cases.c.h" + #line 3284 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2331 "Python/bytecodes.c" + #line 2333 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3291 "Python/generated_cases.c.h" + #line 3293 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3299,7 +3301,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2340 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3320,7 +3322,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3324 "Python/generated_cases.c.h" + #line 3326 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3329,7 +3331,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2363 "Python/bytecodes.c" + #line 2365 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3339,7 +3341,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3343 "Python/generated_cases.c.h" + #line 3345 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3353,7 +3355,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2375 "Python/bytecodes.c" + #line 2377 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3370,7 +3372,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3374 "Python/generated_cases.c.h" + #line 3376 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3384,7 +3386,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2394 "Python/bytecodes.c" + #line 2396 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3394,7 +3396,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3398 "Python/generated_cases.c.h" + #line 3400 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3408,7 +3410,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2406 "Python/bytecodes.c" + #line 2408 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3422,7 +3424,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3426 "Python/generated_cases.c.h" + #line 3428 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3431,16 +3433,16 @@ } TARGET(KW_NAMES) { - #line 2422 "Python/bytecodes.c" + #line 2424 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3439 "Python/generated_cases.c.h" + #line 3441 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2428 "Python/bytecodes.c" + #line 2430 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3453,7 +3455,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3457 "Python/generated_cases.c.h" + #line 3459 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3463,7 +3465,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2473 "Python/bytecodes.c" + #line 2475 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3544,7 +3546,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3548 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3556,7 +3558,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2560 "Python/bytecodes.c" + #line 2562 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3566,7 +3568,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3570 "Python/generated_cases.c.h" + #line 3572 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3575,7 +3577,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2572 "Python/bytecodes.c" + #line 2574 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3600,7 +3602,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3604 "Python/generated_cases.c.h" + #line 3606 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3609,7 +3611,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2599 "Python/bytecodes.c" + #line 2601 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3639,7 +3641,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3643 "Python/generated_cases.c.h" + #line 3645 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3647,7 +3649,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2631 "Python/bytecodes.c" + #line 2633 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3657,7 +3659,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3661 "Python/generated_cases.c.h" + #line 3663 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3670,7 +3672,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2643 "Python/bytecodes.c" + #line 2645 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3681,7 +3683,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3685 "Python/generated_cases.c.h" + #line 3687 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3695,7 +3697,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2657 "Python/bytecodes.c" + #line 2659 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3706,7 +3708,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3710 "Python/generated_cases.c.h" + #line 3712 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3720,7 +3722,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2671 "Python/bytecodes.c" + #line 2673 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3742,7 +3744,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3746 "Python/generated_cases.c.h" + #line 3748 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3756,7 +3758,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2696 "Python/bytecodes.c" + #line 2698 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3784,7 +3786,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3788 "Python/generated_cases.c.h" + #line 3790 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3798,7 +3800,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2727 "Python/bytecodes.c" + #line 2729 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3830,7 +3832,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3834 "Python/generated_cases.c.h" + #line 3836 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3844,7 +3846,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2762 "Python/bytecodes.c" + #line 2764 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3876,7 +3878,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3880 "Python/generated_cases.c.h" + #line 3882 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3890,7 +3892,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2797 "Python/bytecodes.c" + #line 2799 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3915,7 +3917,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3919 "Python/generated_cases.c.h" + #line 3921 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3928,7 +3930,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2824 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3955,7 +3957,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3959 "Python/generated_cases.c.h" + #line 3961 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3967,7 +3969,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2854 "Python/bytecodes.c" + #line 2856 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3985,14 +3987,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3989 "Python/generated_cases.c.h" + #line 3991 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2874 "Python/bytecodes.c" + #line 2876 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4023,7 +4025,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4027 "Python/generated_cases.c.h" + #line 4029 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4036,7 +4038,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2908 "Python/bytecodes.c" + #line 2910 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4065,7 +4067,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4069 "Python/generated_cases.c.h" + #line 4071 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4078,7 +4080,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2940 "Python/bytecodes.c" + #line 2942 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4107,7 +4109,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4111 "Python/generated_cases.c.h" + #line 4113 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4120,7 +4122,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2972 "Python/bytecodes.c" + #line 2974 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4148,7 +4150,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4152 "Python/generated_cases.c.h" + #line 4154 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4158,9 +4160,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3003 "Python/bytecodes.c" + #line 3005 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4164 "Python/generated_cases.c.h" + #line 4166 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4169,7 +4171,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3007 "Python/bytecodes.c" + #line 3009 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4212,14 +4214,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4216 "Python/generated_cases.c.h" + #line 4218 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3050 "Python/bytecodes.c" + #line 3052 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4223 "Python/generated_cases.c.h" + #line 4225 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4234,7 +4236,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3060 "Python/bytecodes.c" + #line 3062 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4263,14 +4265,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4267 "Python/generated_cases.c.h" + #line 4269 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3091 "Python/bytecodes.c" + #line 3093 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4291,7 +4293,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4295 "Python/generated_cases.c.h" + #line 4297 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4299,15 +4301,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3114 "Python/bytecodes.c" + #line 3116 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4305 "Python/generated_cases.c.h" + #line 4307 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3116 "Python/bytecodes.c" + #line 3118 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4311 "Python/generated_cases.c.h" + #line 4313 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4318,7 +4320,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3120 "Python/bytecodes.c" + #line 3122 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4353,7 +4355,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4357 "Python/generated_cases.c.h" + #line 4359 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4362,10 +4364,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3157 "Python/bytecodes.c" + #line 3159 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4369 "Python/generated_cases.c.h" + #line 4371 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4377,7 +4379,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3162 "Python/bytecodes.c" + #line 3164 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4392,12 +4394,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4396 "Python/generated_cases.c.h" + #line 4398 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3177 "Python/bytecodes.c" + #line 3179 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4401 "Python/generated_cases.c.h" + #line 4403 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4407,16 +4409,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3182 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" assert(oparg >= 2); - #line 4413 "Python/generated_cases.c.h" + #line 4415 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3186 "Python/bytecodes.c" + #line 3188 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4436,11 +4438,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4440 "Python/generated_cases.c.h" + #line 4442 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3208 "Python/bytecodes.c" + #line 3210 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4452,20 +4454,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4456 "Python/generated_cases.c.h" + #line 4458 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3223 "Python/bytecodes.c" + #line 3224 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4462 "Python/generated_cases.c.h" + #line 4464 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3228 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4469 "Python/generated_cases.c.h" + #line 4471 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4480,7 +4482,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4484 "Python/generated_cases.c.h" + #line 4486 "Python/generated_cases.c.h" DISPATCH(); } @@ -4494,7 +4496,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4498 "Python/generated_cases.c.h" + #line 4500 "Python/generated_cases.c.h" DISPATCH(); } @@ -4512,7 +4514,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4516 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" DISPATCH(); } @@ -4530,7 +4532,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4534 "Python/generated_cases.c.h" + #line 4536 "Python/generated_cases.c.h" DISPATCH(); } @@ -4541,19 +4543,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4545 "Python/generated_cases.c.h" + #line 4547 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3293 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4552 "Python/generated_cases.c.h" + #line 4554 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3298 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4559 "Python/generated_cases.c.h" + #line 4561 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index d1ec45511d841e..20ebdd8bf52ff7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -11,10 +11,10 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; - case INSTRUMENTED_RESUME: - return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 0; case LOAD_FAST_CHECK: @@ -278,7 +278,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case FOR_ITER: return 1; case INSTRUMENTED_FOR_ITER: - return -1; + return 0; case FOR_ITER_LIST: return 1; case FOR_ITER_TUPLE: @@ -342,7 +342,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return oparg + 2; case INSTRUMENTED_CALL_FUNCTION_EX: - return -1; + return 0; case CALL_FUNCTION_EX: return ((oparg & 1) ? 1 : 0) + 3; case MAKE_FUNCTION: @@ -395,10 +395,10 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; - case INSTRUMENTED_RESUME: - return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 1; case LOAD_FAST_CHECK: @@ -662,7 +662,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case FOR_ITER: return 2; case INSTRUMENTED_FOR_ITER: - return -1; + return 0; case FOR_ITER_LIST: return 2; case FOR_ITER_TUPLE: @@ -726,7 +726,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return 1; case INSTRUMENTED_CALL_FUNCTION_EX: - return -1; + return 0; case CALL_FUNCTION_EX: return 1; case MAKE_FUNCTION: @@ -782,8 +782,8 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[256]; #else const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [NOP] = { true, INSTR_FMT_IX }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB }, [RESUME] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB }, [LOAD_FAST] = { true, INSTR_FMT_IB }, From 43618a9d8f766f64f6a35e4c5a21d55aa9bc7715 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 19:56:45 +0100 Subject: [PATCH 110/116] Keep check-c-globals happy. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index f7e6843c12969f..8745f8abc224fe 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -136,6 +136,7 @@ Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - Python/legacy_tracing.c - _PyLegacyEventHandler_Type - +Objects/object.c - _PyLegacyEventHandler_Type - ##----------------------- From 821ae52abcd44eccae583a647bd4c50c3b7125a7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 02:43:40 +0100 Subject: [PATCH 111/116] Don't crash if locals events aren't set. --- Lib/test/test_monitoring.py | 9 +++++++++ Python/instrumentation.c | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fe93046c8321f9..0e88efa644ba8a 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1020,3 +1020,12 @@ def test_local(self): self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) sys.monitoring.set_local_events(TEST_TOOL2, code, 0) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) + +class TestUninitialized(unittest.TestCase, MonitoringTestBase): + + @staticmethod + def f(): + pass + + def test_get_local_events_uninitialized(self): + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, self.f.__code__), 0) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 436c2b48d178ab..dcb9dc81b19dc0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1793,10 +1793,12 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, return -1; } _PyMonitoringEventSet event_set = 0; - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - _Py_Monitors *m = &((PyCodeObject *)code)->_co_monitoring->local_monitors; - if ((m->tools[e] >> tool_id) & 1) { - event_set |= (1 << e); + _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; + if (data != NULL) { + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((data->local_monitors.tools[e] >> tool_id) & 1) { + event_set |= (1 << e); + } } } return event_set; From 505a08dfa740ef3296e3779e5b469df96c087850 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 03:00:00 +0100 Subject: [PATCH 112/116] Add NO_EVENTS. --- Lib/test/test_monitoring.py | 1 + Python/instrumentation.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0e88efa644ba8a..8679bca3f234ec 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -49,6 +49,7 @@ def test_has_objects(self): m.restart_events m.DISABLE m.MISSING + m.events.NO_EVENTS def test_tool(self): sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") diff --git a/Python/instrumentation.c b/Python/instrumentation.c index dcb9dc81b19dc0..05783ece83b0d0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -4,6 +4,7 @@ #include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" +#include "pycore_long.h" #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode.h" @@ -1974,6 +1975,8 @@ PyObject *_Py_CreateMonitoringObject(void) goto error; } } + err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); + if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); From f63da91d90a9b3e6bf4b6023a68b541c6f11e19b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 05:00:54 +0100 Subject: [PATCH 113/116] Flip order of LINE and INSTRUCTION events. --- Lib/test/test_monitoring.py | 84 +++++++++++++++++++++---------------- Python/instrumentation.c | 61 ++++++++++++++------------- Python/legacy_tracing.c | 19 +++------ 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 8679bca3f234ec..73406f771f5975 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -33,7 +33,7 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 -class MonitoringBaseTest(unittest.TestCase): +class MonitoringBasicTest(unittest.TestCase): def test_has_objects(self): m = sys.monitoring @@ -54,8 +54,6 @@ def test_has_objects(self): def test_tool(self): sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") - sys.monitoring.free_tool_id(TEST_TOOL) - self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) sys.monitoring.set_events(TEST_TOOL, 15) self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) sys.monitoring.set_events(TEST_TOOL, 0) @@ -63,9 +61,33 @@ def test_tool(self): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) + sys.monitoring.free_tool_id(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) -class MonitoringCountTest(unittest.TestCase): +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL2), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL3), None) + sys.monitoring.use_tool_id(TEST_TOOL, "test " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL2, "test2 " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL3, "test3 " + self.__class__.__name__) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL2) + sys.monitoring.free_tool_id(TEST_TOOL3) + + +class MonitoringCountTest(MonitoringTestBase, unittest.TestCase): def check_event_count(self, func, event, expected): @@ -186,18 +208,6 @@ def nested_call(): PY_CALLABLES = (types.FunctionType, types.MethodType) -class MonitoringTestBase: - - def setUp(self): - # Check that a previous test hasn't left monitoring on. - for tool in range(6): - self.assertEqual(sys.monitoring.get_events(tool), 0) - - def tearDown(self): - # Check that test hasn't left monitoring on. - for tool in range(6): - self.assertEqual(sys.monitoring.get_events(tool), 0) - class MonitoringEventsBase(MonitoringTestBase): def gather_events(self, func): @@ -230,7 +240,7 @@ def check_events(self, func, expected=None): self.assertEqual(events, expected) -class MonitoringEventsTest(unittest.TestCase, MonitoringEventsBase): +class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase): def test_just_pass(self): self.check_events(just_pass) @@ -256,7 +266,7 @@ def test_nested_call(self): from test.profilee import testfunc -class SimulateProfileTest(unittest.TestCase, MonitoringEventsBase): +class SimulateProfileTest(MonitoringEventsBase, unittest.TestCase): def test_balanced(self): events = self.gather_events(testfunc) @@ -322,7 +332,7 @@ def __call__(self, code, event): return sys.monitoring.DISABLE -class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): +class MontoringDisableAndRestartTest(MonitoringTestBase, unittest.TestCase): def test_disable(self): try: @@ -363,7 +373,7 @@ def test_restart(self): sys.monitoring.restart_events() -class MultipleMonitorsTest(unittest.TestCase, MonitoringTestBase): +class MultipleMonitorsTest(MonitoringTestBase, unittest.TestCase): def test_two_same(self): try: @@ -481,7 +491,7 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() -class LineMonitoringTest(unittest.TestCase): +class LineMonitoringTest(MonitoringTestBase, unittest.TestCase): def test_lines_single(self): try: @@ -616,7 +626,7 @@ def __init__(self, events): def __call__(self, code, offset, exc): self.events.append(("raise", type(exc))) -class CheckEvents(unittest.TestCase): +class CheckEvents(MonitoringTestBase, unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -795,14 +805,14 @@ def func1(): self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func1', 2), ('line', 'func1', 1), + ('instruction', 'func1', 2), ('instruction', 'func1', 4), - ('instruction', 'func1', 6), ('line', 'func1', 2), + ('instruction', 'func1', 6), ('instruction', 'func1', 8), - ('instruction', 'func1', 10), ('line', 'func1', 3), + ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), ('line', 'check_events', 11)]) @@ -816,17 +826,17 @@ def func2(): self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func2', 2), ('line', 'func2', 1), + ('instruction', 'func2', 2), ('instruction', 'func2', 4), - ('instruction', 'func2', 6), ('line', 'func2', 2), + ('instruction', 'func2', 6), ('instruction', 'func2', 8), ('instruction', 'func2', 28), ('instruction', 'func2', 30), ('instruction', 'func2', 38), - ('instruction', 'func2', 40), ('line', 'func2', 3), + ('instruction', 'func2', 40), ('instruction', 'func2', 42), ('instruction', 'func2', 44), ('line', 'check_events', 11)]) @@ -843,28 +853,28 @@ def func3(): self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func3', 2), ('line', 'func3', 1), - ('instruction', 'func3', 4), + ('instruction', 'func3', 2), ('line', 'func3', 2), + ('instruction', 'func3', 4), ('instruction', 'func3', 6), - ('instruction', 'func3', 8), ('line', 'func3', 3), + ('instruction', 'func3', 8), ('instruction', 'func3', 18), ('instruction', 'func3', 20), - ('instruction', 'func3', 22), ('line', 'func3', 4), - ('instruction', 'func3', 24), + ('instruction', 'func3', 22), ('line', 'func3', 5), + ('instruction', 'func3', 24), ('instruction', 'func3', 26), ('instruction', 'func3', 28), - ('instruction', 'func3', 30), ('line', 'func3', 6), + ('instruction', 'func3', 30), ('instruction', 'func3', 32), ('instruction', 'func3', 34), ('line', 'check_events', 11)]) -class TestInstallIncrementallly(unittest.TestCase): +class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase): def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -930,7 +940,7 @@ def test_instruction_then_line(self): self.check_events(self.func2, recorders = recorders, must_include = self.MUST_INCLUDE_CI) -class TestLocalEvents(unittest.TestCase): +class TestLocalEvents(MonitoringTestBase, unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -999,7 +1009,7 @@ def func3(): ('line', 'func3', 6)]) -class TestSetGetEvents(unittest.TestCase, MonitoringTestBase): +class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): def test_global(self): sys.monitoring.set_events(TEST_TOOL, E.PY_START) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 05783ece83b0d0..f672456439fabb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -247,12 +247,12 @@ instruction_length(PyCodeObject *code, int offset) int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); assert(opcode != RESERVED); - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { opcode = deinstrumented; @@ -516,12 +516,12 @@ sanity_check_instrumentation(PyCodeObject *code) int _Py_GetBaseOpcode(PyCodeObject *code, int i) { int opcode = _PyCode_CODE(code)[i].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[i]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[i].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } CHECK(opcode != INSTRUMENTED_INSTRUCTION); CHECK(opcode != INSTRUMENTED_LINE); int deinstrumented = DE_INSTRUMENT[opcode]; @@ -540,15 +540,15 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; } - int deinstrumented = DE_INSTRUMENT[opcode]; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } + int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented == 0) { return; } @@ -565,10 +565,6 @@ de_instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode != INSTRUMENTED_LINE) { return; } @@ -589,7 +585,12 @@ static void de_instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->op.code; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; + } if (opcode != INSTRUMENTED_INSTRUCTION) { return; } @@ -612,14 +613,14 @@ instrument(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); } @@ -640,10 +641,6 @@ instrument_line(PyCodeObject *code, int i) { uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { return; } @@ -657,7 +654,13 @@ static void instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->op.code; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_INSTRUCTION) { return; } @@ -672,7 +675,7 @@ instrument_per_instruction(PyCodeObject *code, int i) code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); - instr->op.code = INSTRUMENTED_INSTRUCTION; + *opcode_ptr = INSTRUMENTED_INSTRUCTION; } #ifndef NDEBUG @@ -681,12 +684,12 @@ instruction_has_event(PyCodeObject *code, int offset) { _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } return opcode_has_event(opcode); } #endif diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 6e6deea7b818ce..cf345bddda79b0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -422,17 +422,14 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); + uint32_t events = 0; if (tstate->interp->sys_profiling_threads) { - uint32_t events = + events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND); - _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } - else { - _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, 0); - } - return 0; + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } int @@ -513,8 +510,9 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); + uint32_t events = 0; if (tstate->interp->sys_tracing_threads) { - uint32_t events = + events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | @@ -525,11 +523,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); - } - else { - _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, 0); } - - return 0; + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); } From c32434403726a4d6ca63198bfa31210a06a0c5d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 05:31:48 +0100 Subject: [PATCH 114/116] Check tool is in use when setting events. --- Lib/test/test_monitoring.py | 2 ++ Python/instrumentation.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 73406f771f5975..4aad3da61b6bee 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -63,6 +63,8 @@ def test_tool(self): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) sys.monitoring.free_tool_id(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) class MonitoringTestBase: diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f672456439fabb..39a7eaa3bd2d42 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1551,12 +1551,27 @@ set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) } } +static int +check_tool(PyInterpreterState *interp, int tool_id) +{ + if (tool_id < PY_MONITORING_SYS_PROFILE_ID && + interp->monitoring_tool_names[tool_id] == NULL + ) { + PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id); + return -1; + } + return 0; +} + int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { return 0; @@ -1572,6 +1587,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } if (allocate_instrumentation_data(code)) { return -1; } From 168b34ab6f94ccdcc88f0ab7fc6d2b6e0182b74c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 12 Apr 2023 11:02:56 +0100 Subject: [PATCH 115/116] Reset last traced line number when setting frame.f_trace only if set to new value. --- Lib/test/test_sys_settrace.py | 42 +++++++++++++++++++++++++++++++++++ Objects/frameobject.c | 6 +++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 103f41cc36440c..1cb0afc99b978d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2825,5 +2825,47 @@ def test_events(self): (6, 'line')]) +class TestSetLocalTrace(TraceTestCase): + + def test_with_branches(self): + events = [] + + def tracefunc(frame, event, arg): + if frame.f_code.co_name == "func": + frame.f_trace = tracefunc + line = frame.f_lineno - frame.f_code.co_firstlineno + events.append((line, event)) + return tracefunc + def func(arg = 1): + N = 1 + if arg >= 2: # step 1 + not_reached = 3 + else: + reached = 5 + if arg >= 3: # step 3 + not_reached = 7 + else: + reached = 9 + the_end = 10 + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (2, 'line'), + (5, 'line'), + (6, 'line'), + (9, 'line'), + (10, 'line'), + (10, 'return'), + ] + + sys.settrace(tracefunc) + sys._getframe().f_trace = tracefunc + func() + self.assertEqual(events, EXPECTED_EVENTS) + sys.settrace(None) + + + if __name__ == "__main__": unittest.main() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2facffaca66c0b..ef0070199ab2c0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -852,8 +852,10 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) if (v == Py_None) { v = NULL; } - Py_XSETREF(f->f_trace, Py_XNewRef(v)); - f->f_last_traced_line = -1; + if (v != f->f_trace) { + Py_XSETREF(f->f_trace, Py_XNewRef(v)); + f->f_last_traced_line = -1; + } return 0; } From f07a080b07034730dab295dd431f1851d23fa358 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 12 Apr 2023 11:19:31 +0100 Subject: [PATCH 116/116] Tidy up test case. --- Lib/test/test_sys_settrace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 1cb0afc99b978d..980321e169b9e5 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2828,7 +2828,6 @@ def test_events(self): class TestSetLocalTrace(TraceTestCase): def test_with_branches(self): - events = [] def tracefunc(frame, event, arg): if frame.f_code.co_name == "func": @@ -2836,13 +2835,14 @@ def tracefunc(frame, event, arg): line = frame.f_lineno - frame.f_code.co_firstlineno events.append((line, event)) return tracefunc + def func(arg = 1): N = 1 - if arg >= 2: # step 1 + if arg >= 2: not_reached = 3 else: reached = 5 - if arg >= 3: # step 3 + if arg >= 3: not_reached = 7 else: reached = 9 @@ -2859,6 +2859,7 @@ def func(arg = 1): (10, 'return'), ] + events = [] sys.settrace(tracefunc) sys._getframe().f_trace = tracefunc func()