[Python-checkins] [2.7] bpo-10544: Deprecate "yield" in comprehensions and generator expressions in Py3k mode. (GH-4579) (#4676)

Sat Dec 2 14:00:12 EST 2017

https://github.com/python/cpython/commit/65d1887170fb278c10a836e9e4319cae4707f524
commit: 65d1887170fb278c10a836e9e4319cae4707f524
branch: 2.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2017年12月02日T21:00:09+02:00
summary:
[2.7] bpo-10544: Deprecate "yield" in comprehensions and generator expressions in Py3k mode. (GH-4579) (#4676)
files:
A Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst
M Lib/test/support/__init__.py
M Lib/test/test_generators.py
M Lib/test/test_grammar.py
M Python/symtable.c

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 0f0da72b190..cabf548ec38 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -847,8 +847,8 @@ def make_bad_fd():
 file.close()
 unlink(TESTFN)
 
-def check_syntax_error(testcase, statement, lineno=None, offset=None):
- with testcase.assertRaises(SyntaxError) as cm:
+def check_syntax_error(testcase, statement, errtext='', lineno=None, offset=None):
+ with testcase.assertRaisesRegexp(SyntaxError, errtext) as cm:
 compile(statement, '<test string>', 'exec')
 err = cm.exception
 if lineno is not None:
diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index 5e1a3e52d50..0f7bf19abb8 100644
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -1524,13 +1524,7 @@ def printsolution(self, x):
 [None]
 
 
-
-An obscene abuse of a yield expression within a generator expression:
-
->>> list((yield 21) for i in range(4))
-[21, None, 21, None, 21, None, 21, None]
-
-And a more sane, but still weird usage:
+Yield is allowed only in the outermost iterable in generator expression:
 
 >>> def f(): list(i for i in [(yield 26)])
 >>> type(f())
@@ -1571,7 +1565,7 @@ def printsolution(self, x):
 >>> def f(): return lambda x=(yield): 1
 Traceback (most recent call last):
 ...
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[22]>, line 1)
+SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[21]>, line 1)
 
 >>> def f(): x = yield = y
 Traceback (most recent call last):
@@ -1784,7 +1778,7 @@ def printsolution(self, x):
 >>> type(f())
 <type 'generator'>
 
->>> def f(): x=(i for i in (yield) if (yield))
+>>> def f(): x=(i for i in (yield) if i)
 >>> type(f())
 <type 'generator'>
 
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 5f77c1d018b..fc675c35a30 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -493,6 +493,46 @@ def g2(): return 1
 def testYield(self):
 check_syntax_error(self, "class foo:yield 1")
 
+ def test_yield_in_comprehensions(self):
+ # Check yield in comprehensions
+ def g(): [x for x in [(yield 1)]]
+
+ def check(code, warntext):
+ with check_py3k_warnings((warntext, DeprecationWarning)):
+ compile(code, '<test string>', 'exec')
+ if sys.py3kwarning:
+ import warnings
+ with warnings.catch_warnings():
+ warnings.filterwarnings('error', category=DeprecationWarning)
+ with self.assertRaises(SyntaxError) as cm:
+ compile(code, '<test string>', 'exec')
+ self.assertIn(warntext, str(cm.exception))
+
+ check("def g(): [(yield x) for x in ()]",
+ "'yield' inside list comprehension")
+ check("def g(): [x for x in () if not (yield x)]",
+ "'yield' inside list comprehension")
+ check("def g(): [y for x in () for y in [(yield x)]]",
+ "'yield' inside list comprehension")
+ check("def g(): {(yield x) for x in ()}",
+ "'yield' inside set comprehension")
+ check("def g(): {(yield x): x for x in ()}",
+ "'yield' inside dict comprehension")
+ check("def g(): {x: (yield x) for x in ()}",
+ "'yield' inside dict comprehension")
+ check("def g(): ((yield x) for x in ())",
+ "'yield' inside generator expression")
+ with check_py3k_warnings(("'yield' inside list comprehension",
+ DeprecationWarning)):
+ check_syntax_error(self, "class C: [(yield x) for x in ()]")
+ check("class C: ((yield x) for x in ())",
+ "'yield' inside generator expression")
+ with check_py3k_warnings(("'yield' inside list comprehension",
+ DeprecationWarning)):
+ check_syntax_error(self, "[(yield x) for x in ()]")
+ check("((yield x) for x in ())",
+ "'yield' inside generator expression")
+
 def testRaise(self):
 # 'raise' test [',' test]
 try: raise RuntimeError, 'just testing'
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst
new file mode 100644
index 00000000000..d0d3a75c4cd
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst	
@@ -0,0 +1,4 @@
+Yield expressions are now deprecated in comprehensions and generator
+expressions when checking Python 3 compatibility. They are still
+permitted in the definition of the outermost iterable, as that is
+evaluated directly in the enclosing scope.
diff --git a/Python/symtable.c b/Python/symtable.c
index 3b4247b4153..21790b1cd18 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -162,12 +162,14 @@ PyTypeObject PySTEntry_Type = {
 };
 
 static int symtable_analyze(struct symtable *st);
-static int symtable_warn(struct symtable *st, char *msg, int lineno);
+static int symtable_warn(struct symtable *st,
+ PyObject *warn, const char *msg, int lineno);
 static int symtable_enter_block(struct symtable *st, identifier name,
 _Py_block_ty block, void *ast, int lineno);
 static int symtable_exit_block(struct symtable *st, void *ast);
 static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
 static int symtable_visit_expr(struct symtable *st, expr_ty s);
+static int symtable_visit_listcomp(struct symtable *st, expr_ty e);
 static int symtable_visit_genexp(struct symtable *st, expr_ty s);
 static int symtable_visit_setcomp(struct symtable *st, expr_ty e);
 static int symtable_visit_dictcomp(struct symtable *st, expr_ty e);
@@ -796,14 +798,18 @@ symtable_analyze(struct symtable *st)
 
 
 static int
-symtable_warn(struct symtable *st, char *msg, int lineno)
+symtable_warn(struct symtable *st, PyObject *warn, const char *msg, int lineno)
 {
- if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, st->st_filename,
- lineno, NULL, NULL) < 0) {
- if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) {
+ if (lineno < 0) {
+ lineno = st->st_cur->ste_lineno;
+ }
+ if (PyErr_WarnExplicit(warn, msg, st->st_filename, lineno, NULL, NULL) < 0) {
+ if (PyErr_ExceptionMatches(warn)) {
+ /* Replace the warning exception with a SyntaxError
+ to get a more accurate error report */
+ PyErr_Clear();
 PyErr_SetString(PyExc_SyntaxError, msg);
- PyErr_SyntaxLocation(st->st_filename,
- st->st_cur->ste_lineno);
+ PyErr_SyntaxLocation(st->st_filename, lineno);
 }
 return 0;
 }
@@ -1153,7 +1159,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
 PyOS_snprintf(buf, sizeof(buf),
 GLOBAL_AFTER_USE,
 c_name);
- if (!symtable_warn(st, buf, s->lineno))
+ if (!symtable_warn(st, PyExc_SyntaxWarning, buf, s->lineno))
 return 0;
 }
 if (!symtable_add_def(st, name, DEF_GLOBAL))
@@ -1221,8 +1227,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
 VISIT_SEQ(st, expr, e->v.Set.elts);
 break;
 case ListComp_kind:
- VISIT(st, expr, e->v.ListComp.elt);
- VISIT_SEQ(st, comprehension, e->v.ListComp.generators);
+ if (!symtable_visit_listcomp(st, e))
+ return 0;
 break;
 case GeneratorExp_kind:
 if (!symtable_visit_genexp(st, e))
@@ -1420,12 +1426,11 @@ symtable_visit_alias(struct symtable *st, alias_ty a)
 return r;
 }
 else {
- if (st->st_cur->ste_type != ModuleBlock) {
- int lineno = st->st_cur->ste_lineno;
- if (!symtable_warn(st, IMPORT_STAR_WARNING, lineno)) {
- Py_DECREF(store_name);
- return 0;
- }
+ if (st->st_cur->ste_type != ModuleBlock &&
+ !symtable_warn(st, PyExc_SyntaxWarning, IMPORT_STAR_WARNING, -1))
+ {
+ Py_DECREF(store_name);
+ return 0;
 }
 st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR;
 Py_DECREF(store_name);
@@ -1509,7 +1514,10 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
 !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
 return 0;
 }
- st->st_cur->ste_generator = is_generator;
+ /* In order to check for yield expressions under '-3', we clear
+ the generator flag, and restore it at the end */
+ is_generator |= st->st_cur->ste_generator;
+ st->st_cur->ste_generator = 0;
 /* Outermost iter is received as an argument */
 if (!symtable_implicit_arg(st, 0)) {
 symtable_exit_block(st, (void *)e);
@@ -1527,9 +1535,55 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
 if (value)
 VISIT_IN_BLOCK(st, expr, value, (void*)e);
 VISIT_IN_BLOCK(st, expr, elt, (void*)e);
+ if (Py_Py3kWarningFlag && st->st_cur->ste_generator) {
+ const char *msg = (
+ (e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
+ (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
+ "'yield' inside generator expression");
+ if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) {
+ symtable_exit_block(st, (void *)e);
+ return 0;
+ }
+ }
+ st->st_cur->ste_generator |= is_generator;
 return symtable_exit_block(st, (void *)e);
 }
 
+static int
+symtable_visit_listcomp(struct symtable *st, expr_ty e)
+{
+ asdl_seq *generators = e->v.ListComp.generators;
+ int i, is_generator;
+ /* In order to check for yield expressions under '-3', we clear
+ the generator flag, and restore it at the end */
+ is_generator = st->st_cur->ste_generator;
+ st->st_cur->ste_generator = 0;
+ VISIT(st, expr, e->v.ListComp.elt);
+ for (i = 0; i < asdl_seq_LEN(generators); i++) {
+ comprehension_ty lc = (comprehension_ty)asdl_seq_GET(generators, i);
+ VISIT(st, expr, lc->target);
+ if (i == 0 && !st->st_cur->ste_generator) {
+ /* 'yield' in the outermost iterator doesn't cause a warning */
+ VISIT(st, expr, lc->iter);
+ is_generator |= st->st_cur->ste_generator;
+ st->st_cur->ste_generator = 0;
+ }
+ else {
+ VISIT(st, expr, lc->iter);
+ }
+ VISIT_SEQ(st, expr, lc->ifs);
+ }
+
+ if (Py_Py3kWarningFlag && st->st_cur->ste_generator) {
+ const char *msg = "'yield' inside list comprehension";
+ if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) {
+ return 0;
+ }
+ }
+ st->st_cur->ste_generator |= is_generator;
+ return 1;
+}
+
 static int
 symtable_visit_genexp(struct symtable *st, expr_ty e)
 {
