From f012f05a359f9f2a743e8091762b5d370c7e6f92 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sun, 20 Jun 2021 23:15:59 +0200 Subject: [PATCH 1/9] Stackless issue #270: Remove duplicated code Integrate the C-function PyEval_EvalFrameEx_slp into into the C-function slp_eval_frame_value and rename slp_eval_frame_value to PyEval_EvalFrameEx_slp. Adapt the gdb support library and document the change. --- Lib/test/test_gdb.py | 5 +- Python/ceval.c | 184 ++++++++++++---------------------------- Stackless/changelog.txt | 5 ++ Tools/gdb/libpython.py | 4 +- 4 files changed, 67 insertions(+), 131 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 4d1ce4ed96c06d..b22f5f39fd5229 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -741,8 +741,11 @@ def test_down_at_bottom(self): @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") def test_up_at_top(self): 'Verify handling of "py-up" at the top of the stack' + n = 5 + if support.stackless: + n += 1 bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up'] * 5) + cmds_after_breakpoint=['py-up'] * n) self.assertEndsWith(bt, 'Unable to find an older python frame\n') diff --git a/Python/ceval.c b/Python/ceval.c index 1ff3db0c4172a5..9b7d3b3ff69d45 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -673,7 +673,7 @@ do { \ PyObject* _Py_HOT_FUNCTION #ifdef STACKLESS -slp_eval_frame_value(PyFrameObject *f, int throwflag, PyObject *retval) +PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval_arg) #else _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) #endif @@ -686,9 +686,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) int opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ PyObject **fastlocals, **freevars; -#ifndef STACKLESS PyObject *retval = NULL; /* Return value */ -#endif PyThreadState *tstate = _PyThreadState_GET(); PyCodeObject *co; @@ -970,8 +968,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) /* Stackless specific defines start here.. */ #ifdef STACKLESS - int executing = f->f_executing; - #define SLP_CHECK_INTERRUPT() \ if (tstate->st.interrupt && !tstate->curexc_type) { \ if (tstate->st.tick_counter > tstate->st.tick_watermark) { \ @@ -989,6 +985,34 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } \ tstate->st.tick_counter++; + int executing = f->f_executing; + if (executing == SLP_FRAME_EXECUTING_INVALID) { + --tstate->recursion_depth; + return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval_arg); + } else if (f->f_executing != SLP_FRAME_EXECUTING_NO) { + goto slp_setup_completed; + } + + /* Check, if an extension module has changed tstate->interp->eval_frame. + * PEP 523 defines this function pointer as an API to customise the frame + * evaluation. Stackless can not support this API. In order to prevent + * undefined behavior, we terminate the interpreter. + */ + if (tstate->interp->eval_frame != _PyEval_EvalFrameDefault) + Py_FatalError("An extension module has set a custom frame evaluation function (see PEP 523).\n" + "Stackless Python does not support the frame evaluation API defined by PEP 523.\n" + "The programm now terminates to prevent undefined behavior.\n"); + + if (SLP_CSTACK_SAVE_NOW(tstate, f)) + return slp_eval_frame_newstack(f, throwflag, retval_arg); + + /* push frame */ + if (Py_EnterRecursiveCall("")) { + Py_XDECREF(retval_arg); + SLP_STORE_NEXT_FRAME(tstate, f->f_back); + return NULL; + } + #else #define SLP_CHECK_INTERRUPT() ; @@ -997,11 +1021,8 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) /* push frame */ if (Py_EnterRecursiveCall("")) return NULL; +#endif - /* STACKLESS: - * the code starting from here on until the end-marker - * is duplicated below. Keep the two copies in sync! - */ tstate->frame = f; if (tstate->use_tracing) { @@ -1037,10 +1058,10 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } } } - /* STACKLESS: end of duplicated code - */ - -#endif /* #ifdef STACKLESS */ +#ifdef STACKLESS + executing = SLP_FRAME_EXECUTING_NOVAL; +slp_setup_completed: +#endif if (PyDTrace_FUNCTION_ENTRY_ENABLED()) dtrace_function_entry(f); @@ -1086,14 +1107,14 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) lltrace = _PyDict_GetItemId(f->f_globals, &PyId___ltrace__) != NULL; #endif - if (throwflag) { /* support for generator.throw() */ - assert(retval == NULL); /* to prevent reference leaks */ + if (throwflag) /* support for generator.throw() */ goto error; - } - #ifdef STACKLESS assert(f->f_executing == SLP_FRAME_EXECUTING_VALUE); + assert(retval == NULL); + retval = retval_arg; + retval_arg = NULL; switch(executing){ case SLP_FRAME_EXECUTING_NOVAL: /* don't push it, frame ignores value */ @@ -2026,12 +2047,14 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) STACKLESS_ASSERT(); } else { _Py_IDENTIFIER(send); - if (v == Py_None) { + if (v == Py_None) + { STACKLESS_PROPOSE_METHOD(tstate, receiver, tp_iternext); retval = Py_TYPE(receiver)->tp_iternext(receiver); STACKLESS_ASSERT(); } - else { + else + { STACKLESS_PROPOSE_ALL(tstate); retval = _PyObject_CallMethodIdObjArgs(receiver, &PyId_send, v, NULL); STACKLESS_ASSERT(); @@ -2043,7 +2066,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) HANDLE_UNWINDING(SLP_FRAME_EXECUTING_YIELD_FROM, 0, retval); } if (0) { - slp_continue_slp_eval_frame_yield_from: +slp_continue_slp_eval_frame_yield_from: /* Initialize variables */ receiver = TOP(); } @@ -3132,7 +3155,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) HANDLE_UNWINDING(SLP_FRAME_EXECUTING_ITER, 1, next); } if (0) { - slp_continue_slp_eval_frame_iter: +slp_continue_slp_eval_frame_iter: SLP_SET_OPCODE_AND_OPARG(); assert(opcode == FOR_ITER); @@ -3234,7 +3257,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) HANDLE_UNWINDING(SLP_FRAME_EXECUTING_SETUP_WITH, 1, res); } if(0) { - slp_continue_slp_eval_frame_setup_with: +slp_continue_slp_eval_frame_setup_with: SLP_SET_OPCODE_AND_OPARG(); assert(opcode == SETUP_WITH); /* Initialize variables */ @@ -3323,7 +3346,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) HANDLE_UNWINDING(SLP_FRAME_EXECUTING_WITH_CLEANUP, 0, res); } if (0) { - slp_continue_slp_eval_frame_with_cleanup: +slp_continue_slp_eval_frame_with_cleanup: /* Initialize variables */ exc = TOP(); if (exc == NULL) @@ -3549,6 +3572,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); + #ifdef STACKLESS if (STACKLESS_UNWINDING(result)) { (void) POP(); /* top of stack causes a GC related assertion error */ @@ -3798,29 +3822,24 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } /* pop frame */ -/* exit_eval_frame: */ -#ifndef STACKLESS exit_eval_frame: if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(f); Py_LeaveRecursiveCall(); f->f_executing = 0; - tstate->frame = f->f_back; - - return _Py_CheckFunctionResult(NULL, retval, "PyEval_EvalFrameEx"); - -#else - if (PyDTrace_FUNCTION_RETURN_ENABLED()) - dtrace_function_return(f); - Py_LeaveRecursiveCall(); - f->f_executing = 0; +#ifdef STACKLESS SLP_STORE_NEXT_FRAME(tstate, f->f_back); + Py_CLEAR(retval_arg); +#else + tstate->frame = f->f_back; +#endif return _Py_CheckFunctionResult(NULL, retval, "PyEval_EvalFrameEx"); - +#ifdef STACKLESS stackless_interrupt_call: /* interrupted during unwinding */ + assert(retval_arg == NULL); assert(f->f_executing == SLP_FRAME_EXECUTING_VALUE); f->f_executing = SLP_FRAME_EXECUTING_NOVAL; f->f_stacktop = stack_pointer; @@ -4000,98 +4019,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) assert(SLP_CURRENT_FRAME_IS_VALID(tstate)); return retval; } - -PyObject * _Py_HOT_FUNCTION -PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval) -{ - if (f->f_executing == SLP_FRAME_EXECUTING_INVALID) { - PyThreadState *tstate = _PyThreadState_GET(); - --tstate->recursion_depth; - return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval); - } else if (f->f_executing != SLP_FRAME_EXECUTING_NO) { - return slp_eval_frame_value(f, throwflag, retval); - } - - PyThreadState *tstate = _PyThreadState_GET(); - - /* Check, if an extension module has changed tstate->interp->eval_frame. - * PEP 523 defines this function pointer as an API to customise the frame - * evaluation. Stackless can not support this API. In order to prevent - * undefined behavior, we terminate the interpreter. - */ - if (tstate->interp->eval_frame != _PyEval_EvalFrameDefault) - Py_FatalError("An extension module has set a custom frame evaluation function (see PEP 523).\n" - "Stackless Python does not support the frame evaluation API defined by PEP 523.\n" - "The programm now terminates to prevent undefined behavior.\n"); - - /* Start of code, similar to non stackless - * PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - */ - - if (SLP_CSTACK_SAVE_NOW(tstate, f)) - return slp_eval_frame_newstack(f, throwflag, retval); - - /* push frame */ - if (Py_EnterRecursiveCall("")) { - Py_XDECREF(retval); - SLP_STORE_NEXT_FRAME(tstate, f->f_back); - return NULL; - } - - /* STACKLESS: - * the code starting from here on until the end-marker - * is a copy of code above. Keep the two copies in sync! - */ - tstate->frame = f; - - if (tstate->use_tracing) { - if (tstate->c_tracefunc != NULL) { - /* tstate->c_tracefunc, if defined, is a - function that will be called on *every* entry - to a code block. Its return value, if not - None, is a function that will be called at - the start of each executed line of code. - (Actually, the function must return itself - in order to continue tracing.) The trace - functions are called with three arguments: - a pointer to the current frame, a string - indicating why the function is called, and - an argument which depends on the situation. - The global trace function is also called - whenever an exception is detected. */ - if (call_trace_protected(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, f, PyTrace_CALL, Py_None)) { - /* Trace function raised an error */ - goto exit_eval_frame; - } - } - if (tstate->c_profilefunc != NULL) { - /* Similar for c_profilefunc, except it needn't - return itself and isn't called for "line" events */ - if (call_trace_protected(tstate->c_profilefunc, - tstate->c_profileobj, - tstate, f, PyTrace_CALL, Py_None)) { - /* Profile function raised an error */ - goto exit_eval_frame; - } - } - } - /* STACKLESS: end of duplicated code - */ - - - f->f_executing = SLP_FRAME_EXECUTING_NOVAL; - return slp_eval_frame_value(f, throwflag, retval); -exit_eval_frame: - Py_XDECREF(retval); - if (PyDTrace_FUNCTION_RETURN_ENABLED()) - dtrace_function_return(f); - Py_LeaveRecursiveCall(); - f->f_executing = SLP_FRAME_EXECUTING_NO; - SLP_STORE_NEXT_FRAME(tstate, f->f_back); - return NULL; -} #endif /* #ifdef STACKLESS */ @@ -4288,7 +4215,6 @@ _PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, if (f == NULL) { return NULL; } - fastlocals = f->f_localsplus; freevars = f->f_localsplus + co->co_nlocals; @@ -5076,7 +5002,7 @@ PyObject * PyEval_GetGlobals(void) { PyFrameObject *current_frame = PyEval_GetFrame(); -#if 1 && defined STACKLESS +#ifdef STACKLESS if (current_frame == NULL) { PyThreadState *ts = _PyThreadState_GET(); diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index ab010f4d5f38a9..46e95036fdc4de 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -11,6 +11,11 @@ What's New in Stackless 3.X.X? - https://github.com/stackless-dev/stackless/issues/270 Stackless now uses an unmodified PyFrameObject structure. + The internal C-function "slp_eval_frame_value" has been integrated into the + C-function "PyEval_EvalFrameEx_slp". As a consequence a gdb backtrace shows + "PyEval_EvalFrameEx_slp" instead of "slp_eval_frame_value". Because + "PyEval_EvalFrameEx_slp" invokes itself recursively to setup the C-stack, + you may observe two C stack frames for the first Python frame. - https://github.com/stackless-dev/stackless/issues/269 A failure to unpickle a frame could cause a NULL pointer access when diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 4be4b2a527fa15..3b836296330b25 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1536,7 +1536,9 @@ def is_evalframe(self): try: gdb.lookup_type("PyCFrameObject") # it is Stackless Python - EVALFRAMEEX_FUNCTION_NAME = ('slp_eval_frame_value', 'PyEval_EvalFrame_value') + # - 'PyEval_EvalFrameEx_slp': Stackless starting from version 3.8.0a1 + # - 'slp_eval_frame_value': versions released after Dec 2016 + EVALFRAMEEX_FUNCTION_NAME = ('PyEval_EvalFrameEx_slp', 'slp_eval_frame_value') except gdb.error: # regular CPython EVALFRAMEEX_FUNCTION_NAME = (EVALFRAME,) From ccbd75bad7745a11b1f3df2d721ef21eeadc97be Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Mon, 21 Jun 2021 08:28:56 +0200 Subject: [PATCH 2/9] Stackless issue #XXX: Support PEP-523 eval_frame-hook Support the PEP-523 eval_frame-hook for one particular use case: byte code instrumentation. An example is the pydev-debugger. It can use the hook to inject "break-points". This is much more efficient, than traditional trace functions. --- Include/internal/pycore_stackless.h | 3 +++ Python/ceval.c | 22 +++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_stackless.h b/Include/internal/pycore_stackless.h index 8562d915a2ce83..c1cc5e7ea77eb7 100644 --- a/Include/internal/pycore_stackless.h +++ b/Include/internal/pycore_stackless.h @@ -825,6 +825,9 @@ long slp_parse_thread_id(PyObject *thread_id, unsigned long *id); ((frame_)->f_executing >= SLP_FRAME_EXECUTING_VALUE && \ (frame_)->f_executing <= SLP_FRAME_EXECUTING_YIELD_FROM) +/* Frame is executing, ignore value in retval. + * This is used, if the eval_frame hook is in use. */ +#define SLP_FRAME_EXECUTING_HOOK 100 #endif /* #ifdef SLP_BUILD_CORE */ diff --git a/Python/ceval.c b/Python/ceval.c index 9b7d3b3ff69d45..75b792e0fff23e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -989,19 +989,22 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) if (executing == SLP_FRAME_EXECUTING_INVALID) { --tstate->recursion_depth; return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval_arg); - } else if (f->f_executing != SLP_FRAME_EXECUTING_NO) { + } else if (f->f_executing != SLP_FRAME_EXECUTING_NO && + f->f_executing != SLP_FRAME_EXECUTING_HOOK) { goto slp_setup_completed; } /* Check, if an extension module has changed tstate->interp->eval_frame. * PEP 523 defines this function pointer as an API to customise the frame - * evaluation. Stackless can not support this API. In order to prevent - * undefined behavior, we terminate the interpreter. + * evaluation. Stackless support this API and calls it, if it starts the + * evaluation of a frame. */ - if (tstate->interp->eval_frame != _PyEval_EvalFrameDefault) - Py_FatalError("An extension module has set a custom frame evaluation function (see PEP 523).\n" - "Stackless Python does not support the frame evaluation API defined by PEP 523.\n" - "The programm now terminates to prevent undefined behavior.\n"); + if (f->f_executing == SLP_FRAME_EXECUTING_NO && + tstate->interp->eval_frame != _PyEval_EvalFrameDefault) { + Py_XDECREF(retval_arg); + f->f_executing = SLP_FRAME_EXECUTING_HOOK; + return PyEval_EvalFrameEx(f, throwflag); + } if (SLP_CSTACK_SAVE_NOW(tstate, f)) return slp_eval_frame_newstack(f, throwflag, retval_arg); @@ -3966,6 +3969,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } if (PyFrame_Check(f)) { if (!(f->f_executing == SLP_FRAME_EXECUTING_NO || + f->f_executing == SLP_FRAME_EXECUTING_HOOK || SLP_FRAME_IS_EXECUTING(f))) { PyErr_BadInternalCall(); return NULL; @@ -3987,6 +3991,10 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) retval = Py_None; } + if (f->f_executing == SLP_FRAME_EXECUTING_HOOK) + /* Used, if a eval_frame-hook is installed */ + return PyEval_EvalFrameEx_slp(f, throwflag, retval); + /* test, if the stackless system has been initialized. */ if (tstate->st.main == NULL) { /* Call from extern. Same logic as PyStackless_Call_Main */ From d1e1c03dfe3e8e66412c4db5e17766d12819b47d Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Tue, 22 Jun 2021 17:23:19 +0200 Subject: [PATCH 3/9] Stackless issue #275: Support PEP-523 eval_frame-hook Support the PEP-523 eval_frame-hook for one particular use case: byte code instrumentation. An example is the pydev-debugger. It can use the hook to inject "break-points". This is much more efficient, than traditional trace functions. Improve the implementation and add a test case. --- Python/ceval.c | 34 +++++++++++-- Stackless/module/_teststackless.c | 83 +++++++++++++++++++++++++++++++ Stackless/unittests/test_capi.py | 72 +++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 75b792e0fff23e..f5c9aca94193ee 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -989,8 +989,8 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) if (executing == SLP_FRAME_EXECUTING_INVALID) { --tstate->recursion_depth; return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval_arg); - } else if (f->f_executing != SLP_FRAME_EXECUTING_NO && - f->f_executing != SLP_FRAME_EXECUTING_HOOK) { + } else if (executing != SLP_FRAME_EXECUTING_NO && + executing != SLP_FRAME_EXECUTING_HOOK) { goto slp_setup_completed; } @@ -999,11 +999,37 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) * evaluation. Stackless support this API and calls it, if it starts the * evaluation of a frame. */ - if (f->f_executing == SLP_FRAME_EXECUTING_NO && + if (executing == SLP_FRAME_EXECUTING_NO && tstate->interp->eval_frame != _PyEval_EvalFrameDefault) { Py_XDECREF(retval_arg); f->f_executing = SLP_FRAME_EXECUTING_HOOK; - return PyEval_EvalFrameEx(f, throwflag); + retval = PyEval_EvalFrameEx(f, throwflag); + /* There are two possibilities: + * - Either the hook-functions delegates to _PyEval_EvalFrameDefault + * Then the frame transfer protocol is observed, SLP_STORE_NEXT_FRAME(tstate, f->f_back) + * has been called. + * - Or the the hook function does not call _PyEval_EvalFrameDefault + * Then the frame transfer protocol is (probably) violated and the code just + * assigned tstate->frame = f->f_back. + * + * To distinguish both cases, we look a f->f_executing. If the value is still + * SLP_FRAME_EXECUTING_HOOK, then _PyEval_EvalFrameDefault wasn't called for frame f. + */ + if (f->f_executing != SLP_FRAME_EXECUTING_HOOK) + /* _PyEval_EvalFrameDefault was called */ + return retval; + /* Try to repair the frame reference count. + * It is possible in case of a simple tstate->frame = f->f_back */ + if (tstate->frame == f->f_back) { + SLP_STORE_NEXT_FRAME(tstate, f->f_back); + return retval; + } + /* Game over */ + Py_FatalError("An extension module has set a custom frame evaluation function (see PEP 523).\n" + "Stackless Python does not completely support the frame evaluation API defined by PEP 523.\n" + "The programm now terminates to prevent undefined behavior.\n"); + } else if (executing == SLP_FRAME_EXECUTING_HOOK) { + executing = f->f_executing = SLP_FRAME_EXECUTING_NO; } if (SLP_CSTACK_SAVE_NOW(tstate, f)) diff --git a/Stackless/module/_teststackless.c b/Stackless/module/_teststackless.c index 0f650d2dd95724..9944fab89511de 100644 --- a/Stackless/module/_teststackless.c +++ b/Stackless/module/_teststackless.c @@ -485,6 +485,87 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec return result; } +static PyObject *pep523_frame_hook_throw = NULL; +static PyObject *pep523_frame_hook_store_args = NULL; +static PyObject *pep523_frame_hook_result = NULL; + +static PyObject * +pep523_frame_hook_eval_frame(PyFrameObject *f, int throwflag) { + PyThreadState *tstate = PyThreadState_GET(); + if (pep523_frame_hook_store_args) { + PyObject *ret = PyObject_CallFunction(pep523_frame_hook_store_args, "((Oi))", (PyObject *)f, throwflag); + if (!ret) { + tstate->frame = f->f_back; + return NULL; + } + Py_DECREF(ret); + } + if (pep523_frame_hook_throw) { + PyErr_SetObject((PyObject *)Py_TYPE(pep523_frame_hook_throw), pep523_frame_hook_throw); + Py_CLEAR(pep523_frame_hook_throw); + tstate->frame = f->f_back; + return NULL; + } + PyObject *retval = _PyEval_EvalFrameDefault(f, throwflag); + + /* After _PyEval_EvalFrameDefault the current frame is invalid. + * We must not enter the interpreter and we can't use the frame transfer macros, + * because they are a Py_BUILD_CORE API only. + * In practice entering the interpreter would work. Only if you compile + * Stackless with SLP_WITH_FRAME_REF_DEBUG defined you get assertion failures. + */ + if (retval != NULL ) { + Py_INCREF(retval); + Py_XSETREF(pep523_frame_hook_result, retval); + } else { + Py_INCREF(Py_NotImplemented); /* anything else but None */ + Py_XSETREF(pep523_frame_hook_result, Py_NotImplemented); + } + return retval; +} + +PyDoc_STRVAR(test_install_PEP523_eval_frame_hook__doc__, + "test_install_PEP523_eval_frame_hook(*, store_args, store_result, reset=False) -- a test function.\n\ +This function tests the PEP-523 eval_frame-hook. Usually it is not used by Stackless Python."); + +static PyObject* test_install_PEP523_eval_frame_hook(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = { "throw", "store_args", "reset", NULL }; + PyThreadState *tstate = PyThreadState_GET(); + PyObject *throw = NULL; + PyObject *store_args = NULL; + PyObject *reset = Py_False; + PyObject *retval; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$O!OO!:test_install_PEP523_eval_frame_hook", kwlist, + PyExc_Exception, &throw, &store_args, &PyBool_Type, &reset)) + return NULL; + + if (PyObject_IsTrue(reset)) { + tstate->interp->eval_frame = _PyEval_EvalFrameDefault; + Py_CLEAR(pep523_frame_hook_throw); + Py_CLEAR(pep523_frame_hook_store_args); + } else { + + if (throw) { + Py_INCREF(throw); + Py_XSETREF(pep523_frame_hook_throw, throw); + } + if (store_args) { + Py_INCREF(store_args); + Py_XSETREF(pep523_frame_hook_store_args, store_args); + } + + tstate->interp->eval_frame = pep523_frame_hook_eval_frame; + } + retval = pep523_frame_hook_result; + pep523_frame_hook_result = NULL; + if (!retval) { + retval = Py_Ellipsis; /* just something different from None and NotImplemented */ + Py_INCREF(retval); + } + return retval; +} + /* ---------- */ @@ -500,6 +581,8 @@ static PyMethodDef _teststackless_methods[] = { test_cstate__doc__ }, { "test_PyEval_EvalFrameEx", (PyCFunction)(void(*)(void))test_PyEval_EvalFrameEx, METH_VARARGS | METH_KEYWORDS, test_PyEval_EvalFrameEx__doc__ }, + { "test_install_PEP523_eval_frame_hook", (PyCFunction)(void(*)(void))test_install_PEP523_eval_frame_hook, METH_VARARGS | METH_KEYWORDS, + test_install_PEP523_eval_frame_hook__doc__ }, {NULL, NULL} /* sentinel */ }; diff --git a/Stackless/unittests/test_capi.py b/Stackless/unittests/test_capi.py index d330a2b61488e2..0f827a17a885cf 100644 --- a/Stackless/unittests/test_capi.py +++ b/Stackless/unittests/test_capi.py @@ -33,6 +33,7 @@ import subprocess import glob import pickle +import types import _teststackless from support import test_main # @UnusedImport from support import (StacklessTestCase, withThreads, require_one_thread, @@ -375,6 +376,77 @@ def _test_soft_switchable_extension(self): sys.stdout.buffer.write(output) +class Test_pep523_frame_hook(StacklessTestCase): + def setUp(self): + super().setUp() + self.is_done = False + def tearDown(self): + super().tearDown() + _teststackless.test_install_PEP523_eval_frame_hook(reset=True) + self.is_done = None + + def record_done(self, fail=False): + self.is_done = sys._getframe() + if fail: + 4711 / 0 + return id(self) + + def test_hook_delegates_to__PyEval_EvalFrameDefault(self): + # test, that the hook function delegates to _PyEval_EvalFrameDefault + args = [] + r1 = _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append) + try: + r2 = self.record_done() + finally: + r3 = _teststackless.test_install_PEP523_eval_frame_hook(reset=True) + + self.assertIs(r1, ...) + self.assertIsInstance(self.is_done, types.FrameType) + self.assertEqual(r2, id(self)) + self.assertEqual(r3, r2) + self.assertListEqual(args, [(self.is_done, 0)]) + + def test_hook_delegates_to__PyEval_EvalFrameDefault_exc(self): + # test, that the hook function delegates to _PyEval_EvalFrameDefault + args = [] + r1 = _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append) + try: + try: + self.record_done(fail=True) + finally: + r2 = _teststackless.test_install_PEP523_eval_frame_hook(reset=True) + except ZeroDivisionError as e: + self.assertEqual(str(e), "division by zero") + else: + self.fail("Expected exception") + + self.assertIs(r1, ...) + self.assertIsInstance(self.is_done, types.FrameType) + self.assertEqual(r2, NotImplemented) + self.assertListEqual(args, [(self.is_done, 0)]) + + def test_hook_raises_exception(self): + # test, that the hook function delegates to _PyEval_EvalFrameDefault + args = [] + _teststackless.test_install_PEP523_eval_frame_hook(store_args=args.append, + throw=ZeroDivisionError("0/0")) + try: + try: + self.record_done() + finally: + r = _teststackless.test_install_PEP523_eval_frame_hook(reset=True) + except ZeroDivisionError as e: + self.assertEqual(str(e), "0/0") + else: + self.fail("Expected exception") + + self.assertIs(self.is_done, False) + self.assertEqual(r, ...) + self.assertEqual(len(args), 1) + self.assertIsInstance(args[0][0], types.FrameType) + self.assertEqual(args[0][1], 0) + + if __name__ == "__main__": if not sys.argv[1:]: sys.argv.append('-v') From 9451c16fd05f9f11af9e04357c9458de646a4ad1 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Tue, 22 Jun 2021 18:52:35 +0200 Subject: [PATCH 4/9] Eliminate one indirection and a variable. Update changelog.txt --- Python/ceval.c | 16 ++++++++++------ Stackless/changelog.txt | 5 +++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index cb0ba1fd325567..3194202db9bbf8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3894,13 +3894,13 @@ handle_unwinding(int lineno, PyFrameObject *f, } PyObject* -PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval_arg) +PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval) { PyThreadState *tstate = _PyThreadState_GET(); int executing = f->f_executing; if (executing == SLP_FRAME_EXECUTING_INVALID) { --tstate->recursion_depth; - return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval_arg); + return slp_cannot_execute((PyCFrameObject *)f, "PyEval_EvalFrameEx_slp", retval); } else if (executing == SLP_FRAME_EXECUTING_NO || executing == SLP_FRAME_EXECUTING_HOOK) { /* Processing of a frame starts here */ @@ -3912,9 +3912,9 @@ PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval_arg) */ if (executing == SLP_FRAME_EXECUTING_NO && tstate->interp->eval_frame != _PyEval_EvalFrameDefault) { - Py_XDECREF(retval_arg); + Py_XDECREF(retval); f->f_executing = SLP_FRAME_EXECUTING_HOOK; - PyObject *retval = PyEval_EvalFrameEx(f, throwflag); + retval = tstate->interp->eval_frame(f, throwflag); /* There are two possibilities: * - Either the hook-functions delegates to _PyEval_EvalFrameDefault * Then the frame transfer protocol is observed, SLP_STORE_NEXT_FRAME(tstate, f->f_back) @@ -3947,10 +3947,14 @@ PyEval_EvalFrameEx_slp(PyFrameObject *f, int throwflag, PyObject *retval_arg) /* Setup the C-stack and recursively call PyEval_EvalFrameEx_slp with the same arguments. * SLP_CSTACK_SAVE_NOW(tstate, f) will be false then. */ - return slp_eval_frame_newstack(f, throwflag, retval_arg); + return slp_eval_frame_newstack(f, throwflag, retval); } } - return slp_eval_frame_value(f, throwflag, retval_arg); + + /* This is the only call of static slp_eval_frame_value. + * An optimizing compiler will eliminate this call + */ + return slp_eval_frame_value(f, throwflag, retval); } diff --git a/Stackless/changelog.txt b/Stackless/changelog.txt index 237b3595eeccc9..f3fa5c7c9b182e 100644 --- a/Stackless/changelog.txt +++ b/Stackless/changelog.txt @@ -9,6 +9,11 @@ What's New in Stackless 3.X.X? *Release date: 20XX-XX-XX* +- https://github.com/stackless-dev/stackless/issues/275 + Add limited support for the eval_frame-hook introduced by PEP-523. + Stackless can't fully support all use-cases. Currently the hook can be used + by a debugger (i.e. pydevd) to modify the Python byte-code. + - https://github.com/stackless-dev/stackless/issues/270 Stackless now uses an unmodified PyFrameObject structure. Stackless now stores more state information in the field f->f_executing than C-Python. From 00f9eba291cbb7e379bac3ce98f5e68dce27eb8c Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Tue, 22 Jun 2021 19:11:58 +0200 Subject: [PATCH 5/9] revert gdb related changes --- Lib/test/test_gdb.py | 5 +---- Tools/gdb/libpython.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index b22f5f39fd5229..4d1ce4ed96c06d 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -741,11 +741,8 @@ def test_down_at_bottom(self): @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") def test_up_at_top(self): 'Verify handling of "py-up" at the top of the stack' - n = 5 - if support.stackless: - n += 1 bt = self.get_stack_trace(script=self.get_sample_script(), - cmds_after_breakpoint=['py-up'] * n) + cmds_after_breakpoint=['py-up'] * 5) self.assertEndsWith(bt, 'Unable to find an older python frame\n') diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 3b836296330b25..4be4b2a527fa15 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1536,9 +1536,7 @@ def is_evalframe(self): try: gdb.lookup_type("PyCFrameObject") # it is Stackless Python - # - 'PyEval_EvalFrameEx_slp': Stackless starting from version 3.8.0a1 - # - 'slp_eval_frame_value': versions released after Dec 2016 - EVALFRAMEEX_FUNCTION_NAME = ('PyEval_EvalFrameEx_slp', 'slp_eval_frame_value') + EVALFRAMEEX_FUNCTION_NAME = ('slp_eval_frame_value', 'PyEval_EvalFrame_value') except gdb.error: # regular CPython EVALFRAMEEX_FUNCTION_NAME = (EVALFRAME,) From a5921fc31fcefdd861a324719d2f3cf94f7fa2bb Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Fri, 16 Jul 2021 21:03:45 +0200 Subject: [PATCH 6/9] Add a missing #include --- Stackless/module/_teststackless.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Stackless/module/_teststackless.c b/Stackless/module/_teststackless.c index 9944fab89511de..b87e4d59ff5852 100644 --- a/Stackless/module/_teststackless.c +++ b/Stackless/module/_teststackless.c @@ -524,6 +524,11 @@ pep523_frame_hook_eval_frame(PyFrameObject *f, int throwflag) { return retval; } +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "pycore_pystate.h" + PyDoc_STRVAR(test_install_PEP523_eval_frame_hook__doc__, "test_install_PEP523_eval_frame_hook(*, store_args, store_result, reset=False) -- a test function.\n\ This function tests the PEP-523 eval_frame-hook. Usually it is not used by Stackless Python."); From 4ae2923a854b5c0e073a7ee6705117c49515c498 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sat, 17 Jul 2021 22:48:37 +0200 Subject: [PATCH 7/9] Remove a redundant define --- Stackless/module/_teststackless.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Stackless/module/_teststackless.c b/Stackless/module/_teststackless.c index b87e4d59ff5852..27bb5824cb7a1c 100644 --- a/Stackless/module/_teststackless.c +++ b/Stackless/module/_teststackless.c @@ -524,9 +524,9 @@ pep523_frame_hook_eval_frame(PyFrameObject *f, int throwflag) { return retval; } -#ifndef Py_BUILD_CORE -#define Py_BUILD_CORE -#endif +/* + * The code below uses Python internal APIs + */ #include "pycore_pystate.h" PyDoc_STRVAR(test_install_PEP523_eval_frame_hook__doc__, From 09fbce8882ec933795adaad351caeaecd73d116b Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sat, 7 Aug 2021 23:02:54 +0200 Subject: [PATCH 8/9] Apply fix from 0fe12a6a3d to the new constant --- Include/internal/pycore_stackless.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_stackless.h b/Include/internal/pycore_stackless.h index 7fd1f44db3034b..3f2f49bb51891e 100644 --- a/Include/internal/pycore_stackless.h +++ b/Include/internal/pycore_stackless.h @@ -816,7 +816,7 @@ long slp_parse_thread_id(PyObject *thread_id, unsigned long *id); /* Frame is executing, ignore value in retval. * This is used, if the eval_frame hook is in use. */ -#define SLP_FRAME_EXECUTING_HOOK 100 +#define SLP_FRAME_EXECUTING_HOOK ((char)100) /* Defined in slp_transfer.c */ int From c84b2c3b7107687d8fc123df9cc44ba8618a4e42 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sat, 7 Aug 2021 23:49:23 +0200 Subject: [PATCH 9/9] Ignore any frames from audit callback functions in our test case. --- Stackless/unittests/test_capi.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Stackless/unittests/test_capi.py b/Stackless/unittests/test_capi.py index 0f827a17a885cf..4020f198c2562a 100644 --- a/Stackless/unittests/test_capi.py +++ b/Stackless/unittests/test_capi.py @@ -380,6 +380,7 @@ class Test_pep523_frame_hook(StacklessTestCase): def setUp(self): super().setUp() self.is_done = False + def tearDown(self): super().tearDown() _teststackless.test_install_PEP523_eval_frame_hook(reset=True) @@ -391,6 +392,11 @@ def record_done(self, fail=False): 4711 / 0 return id(self) + def filter_audit_frames(self, l): + # the test runner installs an audit callback. We have no control + # over this callback, but we can observe its frames. + return list(i for i in l if "audit" not in i[0].f_code.co_name) + def test_hook_delegates_to__PyEval_EvalFrameDefault(self): # test, that the hook function delegates to _PyEval_EvalFrameDefault args = [] @@ -404,6 +410,7 @@ def test_hook_delegates_to__PyEval_EvalFrameDefault(self): self.assertIsInstance(self.is_done, types.FrameType) self.assertEqual(r2, id(self)) self.assertEqual(r3, r2) + args = self.filter_audit_frames(args) self.assertListEqual(args, [(self.is_done, 0)]) def test_hook_delegates_to__PyEval_EvalFrameDefault_exc(self): @@ -423,6 +430,7 @@ def test_hook_delegates_to__PyEval_EvalFrameDefault_exc(self): self.assertIs(r1, ...) self.assertIsInstance(self.is_done, types.FrameType) self.assertEqual(r2, NotImplemented) + args = self.filter_audit_frames(args) self.assertListEqual(args, [(self.is_done, 0)]) def test_hook_raises_exception(self):