Stackless issue #166: Fix PyEval_EvalFrameEx()

Support using PyEval_EvalFrameEx() without a main tasklet. And fix a
potential reference counting problem.

(cherry picked from commit 187184b)

Author: Anselm Kruis

Python/ceval.c

@@ -4104,6 +4104,48 @@ slp_eval_frame_with_cleanup(PyFrameObject *f, int throwflag, PyObject *retval)
     return r;
 }
 
+static PyObject *
+run_frame_dispatch(PyFrameObject *f, int exc, PyObject *retval)
+{
+    PyThreadState *ts = PyThreadState_GET();
+    PyCFrameObject *cf = (PyCFrameObject*) f;
+    PyFrameObject *f_back;
+    int done = cf->i;
+
+    SLP_SET_CURRENT_FRAME(ts, f);
+
+    if (retval == NULL || done)
+        goto exit_run_frame_dispatch;
+
+    Py_DECREF(retval);
+
+    f = (PyFrameObject *)cf->ob1;
+    Py_INCREF(f);  /* cf->ob1 is just a borrowed ref */
+    f_back = (PyFrameObject *)cf->ob2;
+    Py_XINCREF(f_back);  /* cf->ob2 is just a borrowed ref */
+    if (cf->n) {  /* cf->n is throwflag */
+        retval = NULL;
+        slp_bomb_explode(cf->ob3); /* slp_bomb_explode steals the ref to the bomb */
+        cf->ob3 = NULL;
+    } else {
+        retval = cf->ob3; /* cf->ob3 is just a borrowed ref */
+        Py_XINCREF(retval);
+    }
+    retval = slp_frame_dispatch(f, f_back, cf->n, retval);
+    assert(!STACKLESS_UNWINDING(retval));
+    assert(SLP_CURRENT_FRAME_IS_VALID(ts));
+
+    cf->i = 1; /* mark ourself as done */
+
+    Py_DECREF(f);
+    Py_XDECREF(f_back);
+
+    /* pop frame */
+exit_run_frame_dispatch:
+    SLP_STORE_NEXT_FRAME(ts, cf->f_back);
+    return retval;
+}
+
 PyObject *
 _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
 {
@@ -4113,19 +4155,11 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
      */
     PyThreadState *tstate = PyThreadState_GET();
     PyObject * retval = NULL;
+    PyFrameObject * f_back;
 
     if (f == NULL)
         return NULL;
-    /* make sure that the Stackless system has been initialized. */
-    assert(tstate->st.main && tstate->st.current);
-    /* sanity check. */
-    assert(SLP_CURRENT_FRAME_IS_VALID(tstate));
-    if (!throwflag) {
-        /* If throwflag is true, retval must be NULL. Otherwise it must be non-NULL.
-         */
-        Py_INCREF(Py_None);
-        retval = Py_None;
-    }
+
     if (PyFrame_Check(f) && f->f_execute == NULL) {
         /* A new frame returned from PyFrame_New() has f->f_execute == NULL.
          */
@@ -4157,7 +4191,42 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
             return NULL;
         }
     }
-    retval = slp_frame_dispatch(f, f->f_back, throwflag, retval);
+
+    if (!throwflag) {
+        /* If throwflag is true, retval must be NULL. Otherwise it must be non-NULL.
+         */
+        Py_INCREF(Py_None);
+        retval = Py_None;
+    }
+
+    /* test, if the stackless system has been initialized. */
+    if (tstate->st.main == NULL) {
+        /* Call from extern. Same logic as PyStackless_Call_Main */
+        PyCFrameObject *cf;
+
+        cf = slp_cframe_new(run_frame_dispatch, 0);
+        if (cf == NULL)
+            return NULL;
+        if (throwflag) {
+            retval = slp_curexc_to_bomb();
+        }
+        Py_INCREF(f);
+        cf->ob1 = (PyObject*)f;
+        Py_XINCREF(f->f_back);
+        cf->ob2 = (PyObject*)f->f_back;
+        cf->ob3 = retval; /* transfer our ref. slp_frame_dispatch steals the ref */
+        cf->n = throwflag;
+        retval = slp_eval_frame((PyFrameObject *) cf);
+        Py_DECREF((PyObject *)cf);
+        return retval;
+    }
+
+    /* sanity check. */
+    assert(SLP_CURRENT_FRAME_IS_VALID(tstate));
+    f_back = f->f_back;
+    Py_XINCREF(f_back);
+    retval = slp_frame_dispatch(f, f_back, throwflag, retval);
+    Py_XDECREF(f_back);
     assert(!STACKLESS_UNWINDING(retval));
     assert(SLP_CURRENT_FRAME_IS_VALID(tstate));
     return retval;

Stackless/module/stacklessmodule.c

@@ -1134,19 +1134,25 @@ by Stackless Python.\n\
 The function creates a frame from code, globals and args and executes the frame.");
 
 static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObject *kwds) {
-    static char *kwlist[] = {"code", "globals", "args", "alloca", NULL};
+    static char *kwlist[] = {"code", "globals", "args", "alloca", "throw", NULL};
     PyThreadState *tstate = PyThreadState_GET();
     PyCodeObject *co;
     PyObject *globals, *co_args = NULL;
     Py_ssize_t alloca_size = 0;
+    PyObject *exc = NULL;
     PyFrameObject *f;
     PyObject *result = NULL;
     void *p;
     Py_ssize_t na;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!n:test_PyEval_EvalFrameEx", kwlist,
-            &PyCode_Type, &co, &PyDict_Type, &globals, &PyTuple_Type, &co_args, &alloca_size))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!nO:test_PyEval_EvalFrameEx", kwlist,
+            &PyCode_Type, &co, &PyDict_Type, &globals, &PyTuple_Type, &co_args, &alloca_size,
+            &exc))
         return NULL;
+    if (exc && !PyExceptionInstance_Check(exc)) {
+        PyErr_SetString(PyExc_TypeError, "exc must be an exception instance");
+        return NULL;
+    }
     p = alloca(alloca_size);
     assert(globals != NULL);
     assert(tstate != NULL);
@@ -1184,7 +1190,10 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec
             Py_SETREF(fastlocals[0], PyLong_FromVoidPtr(p));
         }
     }
-    result = PyEval_EvalFrameEx(f,0);
+    if (exc) {
+        PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
+    }
+    result = PyEval_EvalFrameEx(f, exc != NULL);
     /* result = Py_None; Py_INCREF(Py_None); */
 exit:
     ++tstate->recursion_depth;

Stackless/unittests/test_capi.py

@@ -23,12 +23,23 @@
 from __future__ import print_function, absolute_import, division
 
 import inspect
+import io
 import stackless
 import sys
+import time
+import traceback
 import unittest
 
 from support import test_main  # @UnusedImport
-from support import StacklessTestCase
+from support import StacklessTestCase, withThreads, require_one_thread
+
+if withThreads:
+    try:
+        import thread as _thread
+    except ImportError:
+        import _thread
+else:
+    _thread = None
 
 
 class Test_PyEval_EvalFrameEx(StacklessTestCase):
@@ -93,6 +104,63 @@ def f():
         self.assertTrue(self.other_done)
         self.assertEqual(result, 666)
 
+    def test_throw(self):
+        def f():
+            raise RuntimeError("must not be called")
+
+        try:
+            exc = ZeroDivisionError("test")
+            stackless.test_PyEval_EvalFrameEx(f.__code__, f.__globals__, (), throw=exc)
+        except ZeroDivisionError:
+            e, tb = sys.exc_info()[1:]
+            self.assertIs(e, exc)
+            self.assertEqual(traceback.extract_tb(tb)[-1][2], f.__name__)
+        else:
+            self.fail("expected exception")
+
+    @unittest.skipUnless(withThreads, "test requires threading")
+    def test_no_main_tasklet(self):
+        def f(self, lock):
+            self.thread_done = True
+            lock.release()
+
+        def g(func, *args, **kw):
+            return func(*args, **kw)
+
+        lock = _thread.allocate_lock()
+
+        # first a dry run to check the threading logic
+        lock.acquire()
+        self.thread_done = False
+        _thread.start_new_thread(g, (stackless.test_PyEval_EvalFrameEx, f.__code__, f.__globals__, (self, lock)))
+        lock.acquire()
+        self.assertTrue(self.thread_done)
+        # return  # uncomment to skip the real test
+        # and now the real smoke test: now PyEval_EvalFrameEx gets called without a main tasklet
+        self.thread_done = False
+        _thread.start_new_thread(stackless.test_PyEval_EvalFrameEx, (f.__code__, f.__globals__, (self, lock)))
+        lock.acquire()
+        self.assertTrue(self.thread_done)
+
+    @unittest.skipUnless(withThreads, "test requires threading")
+    @require_one_thread  # we fiddle with sys.stderr
+    def test_throw_no_main_tasklet(self):
+        def f():
+            raise RuntimeError("must not be called")
+
+        exc = ZeroDivisionError("test")
+        saved_sys_stderr = sys.stderr
+        sys.stderr = iobuffer = io.StringIO()
+        try:
+            _thread.start_new_thread(stackless.test_PyEval_EvalFrameEx, (f.__code__, f.__globals__, ()), dict(throw=exc))
+            time.sleep(0.1)
+        finally:
+            sys.stderr = saved_sys_stderr
+        msg = str(iobuffer.getvalue())
+        self.assertNotIn("Pending error while entering Stackless subsystem", msg)
+        self.assertIn(str(exc), msg)
+        #print(msg)
+
 
 if __name__ == "__main__":
     if not sys.argv[1:]: