merge 3.3-slp (Stackless python#119)

Author: Anselm Kruis

Objects/abstract.c

@@ -2054,14 +2054,14 @@ PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw)
 #endif
         Py_LeaveRecursiveCall();
 #ifdef NDEBUG
-        if (STACKLESS_RETVAL(result) == NULL && !PyErr_Occurred()) {
+        if (STACKLESS_RETVAL(PyThreadState_GET(), result) == NULL && !PyErr_Occurred()) {
             PyErr_SetString(
                 PyExc_SystemError,
                 "NULL result without error in PyObject_Call");
         }
 #else
-        assert((STACKLESS_RETVAL(result) != NULL && !PyErr_Occurred())
-                || (STACKLESS_RETVAL(result) == NULL && PyErr_Occurred()));
+        assert((STACKLESS_RETVAL(PyThreadState_GET(), result) != NULL && !PyErr_Occurred())
+                || (STACKLESS_RETVAL(PyThreadState_GET(), result) == NULL && PyErr_Occurred()));
 #endif
         return result;
     }
@@ -2681,7 +2681,7 @@ PyIter_Next(PyObject *iter)
     STACKLESS_PROMOTE_METHOD(iter, tp_iternext);
     result = (*iter->ob_type->tp_iternext)(iter);
     STACKLESS_ASSERT();
-    STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(result, NULL);
+    STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(PyThreadState_GET(), result, NULL);
     if (result == NULL &&
         PyErr_Occurred() &&
         PyErr_ExceptionMatches(PyExc_StopIteration))

Objects/typeobject.c

@@ -5460,7 +5460,7 @@ wrap_next(PyObject *self, PyObject *args, void *wrapped)
     STACKLESS_PROMOTE_ALL();
     res = (*func)(self);
     STACKLESS_ASSERT();
-    STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(res, NULL);
+    STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(PyThreadState_GET(), res, NULL);
     if (res == NULL && !PyErr_Occurred())
         PyErr_SetNone(PyExc_StopIteration);
     return res;
@@ -6335,7 +6335,7 @@ slp_tp_init_callback(PyFrameObject *f, int exc, PyObject *retval)
     }
     ts->frame = f;
     Py_DECREF(cf);
-    return STACKLESS_PACK(retval);
+    return STACKLESS_PACK(ts, retval);
 }
 #endif
 
@@ -6366,7 +6366,7 @@ slot_tp_init(PyObject *self, PyObject *args, PyObject *kwds)
 #ifdef STACKLESS
     if (stackless && !STACKLESS_UNWINDING(res)) {
         /* required, because added a C-frame */
-        STACKLESS_PACK(res);
+        STACKLESS_PACK(PyThreadState_GET(), res);
         return STACKLESS_UNWINDING_MAGIC;
     }
     if (STACKLESS_UNWINDING(res)) {

Python/ceval.c

@@ -3838,8 +3838,8 @@ slp_eval_frame_value(PyFrameObject *f, int throwflag, PyObject *retval)
     if (why != WHY_RETURN)
         retval = NULL;
 
-    assert((STACKLESS_RETVAL(retval) != NULL && !PyErr_Occurred())
-            || (STACKLESS_RETVAL(retval) == NULL && PyErr_Occurred()));
+    assert((STACKLESS_RETVAL(tstate, retval) != NULL && !PyErr_Occurred())
+            || (STACKLESS_RETVAL(tstate, retval) == NULL && PyErr_Occurred()));
 
 fast_yield:
     if (co->co_flags & CO_GENERATOR) {
@@ -3936,7 +3936,7 @@ slp_eval_frame_value(PyFrameObject *f, int throwflag, PyObject *retval)
     f->f_lasti = INSTR_OFFSET() - 1;
     if (tstate->frame->f_back != f)
         return retval;
-    STACKLESS_UNPACK(retval);
+    STACKLESS_UNPACK(tstate, retval);
     retval = tstate->frame->f_execute(tstate->frame, 0, retval);
     if (tstate->frame != f) {
         assert(f->f_execute == slp_eval_frame_value || f->f_execute == slp_eval_frame_noval ||
@@ -3946,7 +3946,7 @@ slp_eval_frame_value(PyFrameObject *f, int throwflag, PyObject *retval)
         return retval;
     }
     if (STACKLESS_UNWINDING(retval))
-        STACKLESS_UNPACK(retval);
+        STACKLESS_UNPACK(tstate, retval);
 
     f->f_stacktop = NULL;
     if (f->f_execute == slp_eval_frame_iter) {
@@ -4340,7 +4340,7 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
     retval = Py_None;
     if (stackless) {
         tstate->frame = f;
-        return STACKLESS_PACK(retval);
+        return STACKLESS_PACK(tstate, retval);
     }
     else {
         if (f->f_back != NULL)
@@ -4896,8 +4896,8 @@ PyEval_CallObjectWithKeywords(PyObject *func, PyObject *arg, PyObject *kw)
     STACKLESS_ASSERT();
     Py_DECREF(arg);
 
-    assert((STACKLESS_RETVAL(result) != NULL && !PyErr_Occurred())
-           || (STACKLESS_RETVAL(result) == NULL && PyErr_Occurred()));
+    assert((STACKLESS_RETVAL(PyThreadState_GET(), result) != NULL && !PyErr_Occurred())
+           || (STACKLESS_RETVAL(PyThreadState_GET(), result) == NULL && PyErr_Occurred()));
     return result;
 }
 
@@ -5052,8 +5052,8 @@ call_function(PyObject ***pp_stack, int oparg
         READ_TIMESTAMP(*pintr1);
         Py_DECREF(func);
     }
-    assert((STACKLESS_RETVAL(x) != NULL && !PyErr_Occurred())
-           || (STACKLESS_RETVAL(x) == NULL && PyErr_Occurred()));
+    assert((STACKLESS_RETVAL(PyThreadState_GET(), x) != NULL && !PyErr_Occurred())
+           || (STACKLESS_RETVAL(PyThreadState_GET(), x) == NULL && PyErr_Occurred()));
 
     /* Clear the stack of the function object.  Also removes
        the arguments in case they weren't consumed already
@@ -5065,8 +5065,8 @@ call_function(PyObject ***pp_stack, int oparg
         PCALL(PCALL_POP);
     }
 
-    assert((STACKLESS_RETVAL(x) != NULL && !PyErr_Occurred())
-           || (STACKLESS_RETVAL(x) == NULL && PyErr_Occurred()));
+    assert((STACKLESS_RETVAL(PyThreadState_GET(), x) != NULL && !PyErr_Occurred())
+           || (STACKLESS_RETVAL(PyThreadState_GET(), x) == NULL && PyErr_Occurred()));
     return x;
 }
 
@@ -5120,11 +5120,11 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
         }
 #ifdef STACKLESS
         f->f_execute = PyEval_EvalFrameEx_slp;
-        if (slp_enable_softswitch) {
+        if (STACKLESS_POSSIBLE()) {
             Py_INCREF(Py_None);
             retval = Py_None;
             tstate->frame = f;
-            return STACKLESS_PACK(retval);
+            return STACKLESS_PACK(tstate, retval);
         }
         return slp_eval_frame(f);
 #else
@@ -5366,8 +5366,8 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk)
     Py_XDECREF(callargs);
     Py_XDECREF(kwdict);
     Py_XDECREF(stararg);
-    assert((STACKLESS_RETVAL(result) != NULL && !PyErr_Occurred())
-           || (STACKLESS_RETVAL(result) == NULL && PyErr_Occurred()));
+    assert((STACKLESS_RETVAL(PyThreadState_GET(), result) != NULL && !PyErr_Occurred())
+           || (STACKLESS_RETVAL(PyThreadState_GET(), result) == NULL && PyErr_Occurred()));
     return result;
 }
 

Stackless/changelog.txt

@@ -9,6 +9,11 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://bitbucket.org/stackless-dev/stackless/issues/119
+  Fix a rare bug in the stack unwinding mechanism, that caused a SystemError
+  exception or an assertion violation, if a __del__()-method or a weakref
+  callback runs during stack unwinding.
+
 - https://bitbucket.org/stackless-dev/stackless/issues/115
   Fix an unlikely crash caused by an context manager, which silences an
   exception.

Stackless/core/stackless_impl.h

@@ -117,7 +117,6 @@ PyObject * slp_tp_init_callback(PyFrameObject *f, int exc, PyObject *retval);
 
 typedef struct {
     PyObject_HEAD
-    PyObject *tempval;
 } PyUnwindObject;
 
 PyAPI_DATA(PyUnwindObject *) Py_UnwindToken;
@@ -139,45 +138,36 @@ PyAPI_DATA(PyTypeObject) PyClassMethodDescr_Type;
 PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
 
-/* fast (release) and safe (debug) access to the unwind token and retval */
+/* access to the unwind token and retval */
 
-#ifdef Py_DEBUG
-
-#define STACKLESS_PACK(retval) \
-    (assert(Py_UnwindToken->tempval == NULL), \
-     Py_UnwindToken->tempval = (retval), \
+#define STACKLESS_PACK(tstate, retval) \
+    (assert((tstate)->st.unwinding_retval == NULL), \
+     (tstate)->st.unwinding_retval = (retval), \
      (PyObject *) Py_UnwindToken)
 
-#define STACKLESS_UNPACK(retval) \
+#define STACKLESS_UNPACK(tstate, retval) \
     ((void)(assert(STACKLESS_UNWINDING(retval)), \
-     retval = Py_UnwindToken->tempval, \
-     Py_UnwindToken->tempval = NULL, retval))
-
-#else
-
-#define STACKLESS_PACK(retval) \
-    (Py_UnwindToken->tempval = (retval), \
-     (PyObject *) Py_UnwindToken)
-
-#define STACKLESS_UNPACK(retval) \
-    ((void)(retval = Py_UnwindToken->tempval, retval))
-
-#endif
+     retval = (tstate)->st.unwinding_retval, \
+     (tstate)->st.unwinding_retval = NULL, retval))
 
 #define STACKLESS_UNWINDING(obj) \
     ((PyObject *) (obj) == (PyObject *) Py_UnwindToken)
 
 /* an arbitrary positive number */
 #define STACKLESS_UNWINDING_MAGIC 0x7fedcba9
 
-#define STACKLESS_RETVAL(obj) \
-    (STACKLESS_UNWINDING(obj) ? Py_UnwindToken->tempval : (obj))
+#define STACKLESS_RETVAL(tstate, obj) \
+    (STACKLESS_UNWINDING(obj) ? (tstate)->st.unwinding_retval : (obj))
 
-#define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(obj, val) \
-    assert(!STACKLESS_UNWINDING(obj) || ((Py_UnwindToken->tempval) != (val)))
+#define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(tstate, obj, val) \
+    assert(!STACKLESS_UNWINDING(obj) || (((tstate)->st.unwinding_retval) != (val)))
 
 /* macros for setting/resetting the stackless flag */
 
+#define STACKLESS_POSSIBLE()                                        \
+    (slp_enable_softswitch && \
+    PyThreadState_GET()->st.unwinding_retval == NULL)
+
 #define STACKLESS_GETARG() int stackless = (stackless = slp_try_stackless, \
                            slp_try_stackless = 0, stackless)
 
@@ -199,16 +189,16 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
 #define STACKLESS_PROMOTE_ALL() ((void)(slp_try_stackless = stackless, NULL))
 
-#define STACKLESS_PROPOSE(func) {int stackless = slp_enable_softswitch; \
+#define STACKLESS_PROPOSE(func) {int stackless = STACKLESS_POSSIBLE(); \
                  STACKLESS_PROMOTE(func);}
 
-#define STACKLESS_PROPOSE_FLAG(flag) {int stackless = slp_enable_softswitch; \
+#define STACKLESS_PROPOSE_FLAG(flag) {int stackless = STACKLESS_POSSIBLE(); \
                       STACKLESS_PROMOTE_FLAG(flag);}
 
-#define STACKLESS_PROPOSE_METHOD(obj, meth) {int stackless = slp_enable_softswitch; \
+#define STACKLESS_PROPOSE_METHOD(obj, meth) {int stackless = STACKLESS_POSSIBLE(); \
                  STACKLESS_PROMOTE_METHOD(obj, meth);}
 
-#define STACKLESS_PROPOSE_ALL() slp_try_stackless = slp_enable_softswitch;
+#define STACKLESS_PROPOSE_ALL() slp_try_stackless = STACKLESS_POSSIBLE()
 
 #define STACKLESS_RETRACT() slp_try_stackless = 0;
 
@@ -541,8 +531,8 @@ PyObject * slp_get_channel_callback(void);
 #define STACKLESS_RETRACT() assert(1)
 #define STACKLESS_ASSERT() assert(1)
 
-#define STACKLESS_RETVAL(obj) (obj)
-#define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(obj, val) assert(1)
+#define STACKLESS_RETVAL(tstate, obj) (obj)
+#define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(tstate, obj, val) assert(1)
 
 #define STACKLESS_DECLARE_METHOD(type, meth)
 

Stackless/core/stackless_tstate.h

@@ -24,29 +24,30 @@ typedef struct _sts {
     struct _tasklet *main;
     /* runnable tasklets */
     struct _tasklet *current;
-    int runcount;
 
     /* scheduling */
     long tick_counter;
     long tick_watermark;
     long interval;
     PyObject * (*interrupt) (void);    /* the fast scheduler */
-    /* trap recursive scheduling via callbacks */
-    int schedlock;
-    int runflags;                               /* flags for stackless.run() behaviour */
 #ifdef WITH_THREAD
     struct {
         PyObject *block_lock;                   /* to block the thread */
         int is_blocked;                         /* waiting to be unblocked */
         int is_idle;                            /* unblocked, but waiting for GIL */
     } thread;
 #endif
-    /* number of nested interpreters (1.0/2.0 merge) */
-    int nesting_level;
     PyObject *del_post_switch;                  /* To decref after a switch */
     PyObject *interrupted;                      /* The interrupted tasklet in stackles.run() */
-    int switch_trap;                            /* if non-zero, switching is forbidden */
     PyObject *watchdogs;                        /* the stack of currently running watchdogs */
+    PyObject *unwinding_retval;                 /* The return value during stack unwinding */
+    int runcount;
+    /* trap recursive scheduling via callbacks */
+    int schedlock;
+    int runflags;                               /* flags for stackless.run() behaviour */
+    /* number of nested interpreters (1.0/2.0 merge) */
+    int nesting_level;
+    int switch_trap;                            /* if non-zero, switching is forbidden */
 } PyStacklessState;
 
 /* internal macro to temporarily disable soft interrupts */
@@ -59,20 +60,21 @@ typedef struct _sts {
     tstate->st.serial_last_jump = 0; \
     tstate->st.cstack_base = NULL; \
     tstate->st.cstack_root = NULL; \
+    tstate->st.main = NULL; \
+    tstate->st.current = NULL; \
     tstate->st.tick_counter = 0; \
     tstate->st.tick_watermark = 0; \
     tstate->st.interval = 0; \
     tstate->st.interrupt = NULL; \
-    tstate->st.schedlock = 0; \
-    tstate->st.main = NULL; \
-    tstate->st.current = NULL; \
+    tstate->st.del_post_switch = NULL; \
+    tstate->st.interrupted = NULL; \
+    tstate->st.watchdogs = NULL; \
+    tstate->st.unwinding_retval = NULL; \
     tstate->st.runcount = 0; \
+    tstate->st.schedlock = 0; \
     tstate->st.nesting_level = 0; \
     tstate->st.runflags = 0; \
-    tstate->st.del_post_switch = NULL; \
-    tstate->st.interrupted = NULL; \
-    tstate->st.switch_trap = 0; \
-    tstate->st.watchdogs = NULL;
+    tstate->st.switch_trap = 0;
 
 
 /* note that the scheduler knows how to zap. It checks if it is in charge
@@ -89,7 +91,8 @@ void slp_kill_tasks_with_stacks(struct _ts *tstate);
     Py_CLEAR(tstate->st.initial_stub); \
     Py_CLEAR(tstate->st.del_post_switch); \
     Py_CLEAR(tstate->st.interrupted); \
-    Py_CLEAR(tstate->st.watchdogs);
+    Py_CLEAR(tstate->st.watchdogs); \
+    Py_CLEAR(tstate->st.unwinding_retval);
 
 #ifdef WITH_THREAD
 

Stackless/core/stackless_util.c

@@ -71,7 +71,7 @@ slp_return_wrapper(PyObject *retval)
     if (retval == NULL)
         return -1;
     if (STACKLESS_UNWINDING(retval)) {
-        STACKLESS_UNPACK(retval);
+        STACKLESS_UNPACK(PyThreadState_GET(), retval);
         Py_XDECREF(retval);
         return 1;
     }

Stackless/core/stacklesseval.c

@@ -932,7 +932,7 @@ slp_gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
 
     if (stackless) {
         assert(exc == 0);
-        return STACKLESS_PACK(retval);
+        return STACKLESS_PACK(ts, retval);
     }
     return slp_frame_dispatch(f, stopframe, exc, retval);
 }
@@ -1021,6 +1021,7 @@ unwind_repr(PyObject *op)
 
 /* dummy deallocator, just in case */
 static void unwind_dealloc(PyObject *op) {
+    assert(0);  /*should never be called*/
 }
 
 static PyTypeObject PyUnwindToken_Type = {
@@ -1042,7 +1043,6 @@ static PyTypeObject PyUnwindToken_Type = {
 
 static PyUnwindObject unwind_token = {
     PyObject_HEAD_INIT(&PyUnwindToken_Type)
-    NULL
 };
 
 PyUnwindObject *Py_UnwindToken = &unwind_token;
@@ -1074,7 +1074,7 @@ slp_frame_dispatch(PyFrameObject *f, PyFrameObject *stopframe, int exc, PyObject
     while (1) {
         retval = f->f_execute(f, exc, retval);
         if (STACKLESS_UNWINDING(retval))
-            STACKLESS_UNPACK(retval);
+            STACKLESS_UNPACK(ts, retval);
         /* A soft switch is only complete here */
         Py_CLEAR(ts->st.del_post_switch);
         f = ts->frame;
@@ -1103,7 +1103,7 @@ slp_frame_dispatch_top(PyObject *retval)
 
         retval = f->f_execute(f, 0, retval);
         if (STACKLESS_UNWINDING(retval))
-            STACKLESS_UNPACK(retval);
+            STACKLESS_UNPACK(ts, retval);
         /* A soft switch is only complete here */
         Py_CLEAR(ts->st.del_post_switch);
         f = ts->frame;