[Python-checkins] cpython: perform yield from delegation by repeating YIELD_FROM opcode (closes #14230)
benjamin.peterson
python-checkins at python.org
Thu Mar 15 21:38:28 CET 2012
http://hg.python.org/cpython/rev/72556ff86828
changeset: 75714:72556ff86828
parent: 75712:9e7374779e19
user: Benjamin Peterson <benjamin at python.org>
date: Thu Mar 15 15:37:39 2012 -0500
summary:
perform yield from delegation by repeating YIELD_FROM opcode (closes #14230)
This allows generators that are using yield from to be seen by debuggers. It
also kills the f_yieldfrom field on frame objects.
Patch mostly from Mark Shannon with a few tweaks by me.
files:
Include/frameobject.h | 1 -
Include/genobject.h | 3 +-
Lib/test/test_pep380.py | 23 +++-
Lib/test/test_sys.py | 2 +-
Objects/frameobject.c | 4 -
Objects/genobject.c | 179 +++++++++------------------
Python/ceval.c | 58 +++-----
Python/compile.c | 6 +-
Python/import.c | 3 +-
9 files changed, 113 insertions(+), 166 deletions(-)
diff --git a/Include/frameobject.h b/Include/frameobject.h
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -27,7 +27,6 @@
to the current stack top. */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
- PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */
/* In a generator, we need to be able to swap between the exception
state inside the generator and the exception state of the calling
diff --git a/Include/genobject.h b/Include/genobject.h
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -19,7 +19,7 @@
/* True if generator is being executed. */
char gi_running;
-
+
/* The code object backing the generator */
PyObject *gi_code;
@@ -35,6 +35,7 @@
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **);
+PyObject *_PyGen_Send(PyGenObject *, PyObject *);
#ifdef __cplusplus
}
diff --git a/Lib/test/test_pep380.py b/Lib/test/test_pep380.py
--- a/Lib/test/test_pep380.py
+++ b/Lib/test/test_pep380.py
@@ -10,7 +10,7 @@
import unittest
import io
import sys
-import traceback
+import inspect
import parser
from test.support import captured_stderr
@@ -919,6 +919,27 @@
next(g1)
g1.close()
+ def test_delegator_is_visible_to_debugger(self):
+ def call_stack():
+ return [f[3] for f in inspect.stack()]
+
+ def gen():
+ yield call_stack()
+ yield call_stack()
+ yield call_stack()
+
+ def spam(g):
+ yield from g
+
+ def eggs(g):
+ yield from g
+
+ for stack in spam(gen()):
+ self.assertTrue('spam' in stack)
+
+ for stack in spam(eggs(gen())):
+ self.assertTrue('spam' in stack and 'eggs' in stack)
+
def test_main():
from test import support
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -730,7 +730,7 @@
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
- check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+ check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size(h + '12P'))
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -444,7 +444,6 @@
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
- Py_CLEAR(f->f_yieldfrom);
co = f->f_code;
if (co->co_zombieframe == NULL)
@@ -476,7 +475,6 @@
Py_VISIT(f->f_exc_type);
Py_VISIT(f->f_exc_value);
Py_VISIT(f->f_exc_traceback);
- Py_VISIT(f->f_yieldfrom);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@@ -510,7 +508,6 @@
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
Py_CLEAR(f->f_trace);
- Py_CLEAR(f->f_yieldfrom);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@@ -714,7 +711,6 @@
f->f_lasti = -1;
f->f_lineno = code->co_firstlineno;
f->f_iblock = 0;
- f->f_yieldfrom = NULL;
_PyObject_GC_TRACK(f);
return f;
diff --git a/Objects/genobject.c b/Objects/genobject.c
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -6,7 +6,6 @@
#include "opcode.h"
static PyObject *gen_close(PyGenObject *gen, PyObject *args);
-static void gen_undelegate(PyGenObject *gen);
static int
gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
@@ -41,15 +40,6 @@
PyObject_GC_Del(gen);
}
-static int
-gen_running(PyGenObject *gen)
-{
- if (gen->gi_running) {
- PyErr_SetString(PyExc_ValueError, "generator already executing");
- return 1;
- }
- return 0;
-}
static PyObject *
gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
@@ -58,7 +48,11 @@
PyFrameObject *f = gen->gi_frame;
PyObject *result;
- assert(!gen->gi_running);
+ if (gen->gi_running) {
+ PyErr_SetString(PyExc_ValueError,
+ "generator already executing");
+ return NULL;
+ }
if (f==NULL || f->f_stacktop == NULL) {
/* Only set exception if called from send() */
if (arg && !exc)
@@ -136,45 +130,10 @@
"send(arg) -> send 'arg' into generator,\n\
return next yielded value or raise StopIteration.");
-static PyObject *
-gen_send(PyGenObject *gen, PyObject *arg)
+PyObject *
+_PyGen_Send(PyGenObject *gen, PyObject *arg)
{
- int exc = 0;
- PyObject *ret;
- PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
- if (gen_running(gen))
- return NULL;
- /* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed?
- * Or would it be valid to rely on borrowed references?
- */
- Py_INCREF(arg);
- if (yf) {
- Py_INCREF(yf);
- gen->gi_running = 1;
- if (PyGen_CheckExact(yf)) {
- ret = gen_send((PyGenObject *)yf, arg);
- } else {
- if (arg == Py_None)
- ret = PyIter_Next(yf);
- else
- ret = PyObject_CallMethod(yf, "send", "O", arg);
- }
- gen->gi_running = 0;
- if (ret) {
- Py_DECREF(yf);
- goto done;
- }
- gen_undelegate(gen);
- Py_CLEAR(arg);
- if (PyGen_FetchStopIterationValue(&arg) < 0) {
- exc = 1;
- }
- Py_DECREF(yf);
- }
- ret = gen_send_ex(gen, arg, exc);
-done:
- Py_XDECREF(arg);
- return ret;
+ return gen_send_ex(gen, arg, 0);
}
PyDoc_STRVAR(close_doc,
@@ -186,49 +145,61 @@
*/
static int
-gen_close_iter(PyGenObject *gen, PyObject *yf)
+gen_close_iter(PyObject *yf)
{
PyObject *retval = NULL;
- int err = 0;
-
+
if (PyGen_CheckExact(yf)) {
retval = gen_close((PyGenObject *)yf, NULL);
- if (!retval)
- err = -1;
+ if (retval == NULL)
+ return -1;
} else {
- PyObject *meth;
- gen->gi_running = 1;
- meth = PyObject_GetAttrString(yf, "close");
+ PyObject *meth = PyObject_GetAttrString(yf, "close");
if (meth == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
+ if (!PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_WriteUnraisable(yf);
- }
PyErr_Clear();
} else {
retval = PyObject_CallFunction(meth, "");
Py_DECREF(meth);
- if (!retval)
- err = -1;
+ if (retval == NULL)
+ return -1;
}
- gen->gi_running = 0;
}
Py_XDECREF(retval);
- return err;
-}
+ return 0;
+}
+
+static PyObject *
+gen_yf(PyGenObject *gen)
+{
+ PyObject *yf = NULL;
+ PyFrameObject *f = gen->gi_frame;
+
+ if (f) {
+ PyObject *bytecode = f->f_code->co_code;
+ unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
+
+ if (code[f->f_lasti + 1] != YIELD_FROM)
+ return NULL;
+ yf = f->f_stacktop[-1];
+ Py_INCREF(yf);
+ }
+
+ return yf;
+}
static PyObject *
gen_close(PyGenObject *gen, PyObject *args)
{
PyObject *retval;
- PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
+ PyObject *yf = gen_yf(gen);
int err = 0;
- if (gen_running(gen))
- return NULL;
if (yf) {
- Py_INCREF(yf);
- err = gen_close_iter(gen, yf);
- gen_undelegate(gen);
+ gen->gi_running = 1;
+ err = gen_close_iter(yf);
+ gen->gi_running = 0;
Py_DECREF(yf);
}
if (err == 0)
@@ -241,8 +212,7 @@
return NULL;
}
if (PyErr_ExceptionMatches(PyExc_StopIteration)
- || PyErr_ExceptionMatches(PyExc_GeneratorExit))
- {
+ || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
@@ -323,29 +293,27 @@
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
- PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
+ PyObject *yf = gen_yf(gen);
if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))
return NULL;
- if (gen_running(gen))
- return NULL;
-
if (yf) {
PyObject *ret;
int err;
- Py_INCREF(yf);
if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
- err = gen_close_iter(gen, yf);
+ gen->gi_running = 1;
+ err = gen_close_iter(yf);
+ gen->gi_running = 0;
Py_DECREF(yf);
- gen_undelegate(gen);
if (err < 0)
return gen_send_ex(gen, Py_None, 1);
goto throw_here;
}
- gen->gi_running = 1;
if (PyGen_CheckExact(yf)) {
+ gen->gi_running = 1;
ret = gen_throw((PyGenObject *)yf, args);
+ gen->gi_running = 0;
} else {
PyObject *meth = PyObject_GetAttrString(yf, "throw");
if (meth == NULL) {
@@ -355,18 +323,22 @@
}
PyErr_Clear();
Py_DECREF(yf);
- gen_undelegate(gen);
- gen->gi_running = 0;
goto throw_here;
}
+ gen->gi_running = 1;
ret = PyObject_CallObject(meth, args);
+ gen->gi_running = 0;
Py_DECREF(meth);
}
- gen->gi_running = 0;
Py_DECREF(yf);
if (!ret) {
PyObject *val;
- gen_undelegate(gen);
+ /* Pop subiterator from stack */
+ ret = *(--gen->gi_frame->f_stacktop);
+ assert(ret == yf);
+ Py_DECREF(ret);
+ /* Termination repetition of YIELD_FROM */
+ gen->gi_frame->f_lasti++;
if (PyGen_FetchStopIterationValue(&val) == 0) {
ret = gen_send_ex(gen, val, 0);
Py_DECREF(val);
@@ -441,45 +413,12 @@
{
PyObject *val = NULL;
PyObject *ret;
- int exc = 0;
- PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
- if (gen_running(gen))
- return NULL;
- if (yf) {
- Py_INCREF(yf);
- /* ceval.c ensures that yf is an iterator */
- gen->gi_running = 1;
- ret = Py_TYPE(yf)->tp_iternext(yf);
- gen->gi_running = 0;
- if (ret) {
- Py_DECREF(yf);
- return ret;
- }
- gen_undelegate(gen);
- if (PyGen_FetchStopIterationValue(&val) < 0)
- exc = 1;
- Py_DECREF(yf);
- }
- ret = gen_send_ex(gen, val, exc);
+ ret = gen_send_ex(gen, val, 0);
Py_XDECREF(val);
return ret;
}
/*
- * In certain recursive situations, a generator may lose its frame
- * before we get a chance to clear f_yieldfrom, so we use this
- * helper function.
- */
-
-static void
-gen_undelegate(PyGenObject *gen) {
- if (gen->gi_frame) {
- Py_XDECREF(gen->gi_frame->f_yieldfrom);
- gen->gi_frame->f_yieldfrom = NULL;
- }
-}
-
-/*
* If StopIteration exception is set, fetches its 'value'
* attribute if any, otherwise sets pvalue to None.
*
@@ -492,7 +431,7 @@
PyGen_FetchStopIterationValue(PyObject **pvalue) {
PyObject *et, *ev, *tb;
PyObject *value = NULL;
-
+
if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
PyErr_Fetch(&et, &ev, &tb);
Py_XDECREF(et);
@@ -548,7 +487,7 @@
};
static PyMethodDef gen_methods[] = {
- {"send",(PyCFunction)gen_send, METH_O, send_doc},
+ {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc},
{"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc},
{"close",(PyCFunction)gen_close, METH_NOARGS, close_doc},
{NULL, NULL} /* Sentinel */
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1170,6 +1170,8 @@
f->f_lasti to -1 (i.e. the index *before* the first instruction)
and YIELD_VALUE doesn't fiddle with f_lasti any more. So this
does work. Promise.
+ YIELD_FROM sets f_lasti to itself, in order to repeated yield
+ multiple values.
When the PREDICT() macros are enabled, some opcode pairs follow in
direct succession without updating f->f_lasti. A successful
@@ -1830,49 +1832,35 @@
TARGET(YIELD_FROM)
u = POP();
- x = PyObject_GetIter(u);
+ x = TOP();
+ /* send u to x */
+ if (PyGen_CheckExact(x)) {
+ retval = _PyGen_Send((PyGenObject *)x, u);
+ } else {
+ if (u == Py_None)
+ retval = PyIter_Next(x);
+ else
+ retval = PyObject_CallMethod(x, "send", "O", u);
+ }
Py_DECREF(u);
- if (x == NULL)
- break;
- /* x is now the iterator, make the first next() call */
- retval = (*Py_TYPE(x)->tp_iternext)(x);
if (!retval) {
- PyObject *et, *ev, *tb;
- /* iter may be exhausted */
- Py_CLEAR(x);
- if (PyErr_Occurred() &&
- !PyErr_ExceptionMatches(PyExc_StopIteration)) {
- /* some other exception */
+ PyObject *val;
+ x = POP(); /* Remove iter from stack */
+ Py_DECREF(x);
+ err = PyGen_FetchStopIterationValue(&val);
+ if (err < 0) {
+ x = NULL;
break;
}
- /* try to get return value from exception */
- PyErr_Fetch(&et, &ev, &tb);
- Py_XDECREF(et);
- Py_XDECREF(tb);
- /* u is return value */
- u = NULL;
- if (ev) {
- u = PyObject_GetAttrString(ev, "value");
- Py_DECREF(ev);
- if (u == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
- /* some other exception */
- break;
- }
- PyErr_Clear();
- }
- }
- if (u == NULL) {
- u = Py_None;
- Py_INCREF(u);
- }
- PUSH(u);
+ x = val;
+ PUSH(x);
continue;
}
- /* x is iterator, retval is value to be yielded */
- f->f_yieldfrom = x;
+ /* x remains on stack, retval is value to be yielded */
f->f_stacktop = stack_pointer;
why = WHY_YIELD;
+ /* and repeat... */
+ f->f_lasti--;
goto fast_yield;
TARGET(YIELD_VALUE)
diff --git a/Python/compile.c b/Python/compile.c
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -840,9 +840,9 @@
case IMPORT_STAR:
return -1;
case YIELD_VALUE:
+ return 0;
case YIELD_FROM:
- return 0;
-
+ return -1;
case POP_BLOCK:
return 0;
case POP_EXCEPT:
@@ -3323,6 +3323,8 @@
ADDOP_O(c, LOAD_CONST, Py_None, consts);
}
if (e->kind == YieldFrom_kind) {
+ ADDOP(c, GET_ITER);
+ ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
}
else {
diff --git a/Python/import.c b/Python/import.c
--- a/Python/import.c
+++ b/Python/import.c
@@ -104,7 +104,8 @@
Python 3.2a2 3180 (add DELETE_DEREF)
Python 3.3a0 3190 __class__ super closure changed
Python 3.3a0 3200 (__qualname__ added)
- 3210 (added size modulo 2**32 to the pyc header)
+ Python 3.3a1 3210 (added size modulo 2**32 to the pyc header)
+ 3220 (changed PEP 380 implementation)
*/
/* MAGIC must change whenever the bytecode emitted by the compiler may no
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list