[Python-checkins] [3.7] bpo-36541: lib2to3: Support named assignment expressions (GH-12702) (GH-19317)

Fri Apr 3 15:14:20 EDT 2020

https://github.com/python/cpython/commit/96c5f5a3a3fabf43e8114d0dbc30bed409da1ba6
commit: 96c5f5a3a3fabf43e8114d0dbc30bed409da1ba6
branch: 3.7
author: Tim Hatch <tim at timhatch.com>
committer: GitHub <noreply at github.com>
date: 2020年04月03日T12:14:15-07:00
summary:
[3.7] bpo-36541: lib2to3: Support named assignment expressions (GH-12702) (GH-19317)
lib2to3: Support named assignment expressions (GH-12702)
There are two copies of the grammar -- the one used by Python itself as
Grammar/Grammar, and the one used by lib2to3 which has necessarily diverged at
Lib/lib2to3/Grammar.txt because it needs to support older syntax an we want it
to be reasonable stable to avoid requiring fixer rewrites.
This brings suport for syntax like `if x:= foo():` to match what the live
Python grammar does.
This should've been added at the time of the walrus operator itself, but lib2to3 being
independent is often overlooked. So we do consider this a bugfix rather than enhancement.
(cherry picked from commit 3c3aa4516c70753de06bb142b6793d01330fcf0f)
Co-authored-by: Tim Hatch <tim at timhatch.com>
files:
A Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst
M Lib/lib2to3/Grammar.txt
M Lib/lib2to3/pgen2/grammar.py
M Lib/lib2to3/pgen2/token.py
M Lib/lib2to3/pgen2/tokenize.py
M Lib/lib2to3/tests/test_parser.py

diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index 68b73868b5828..8ce7fd8a89da6 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -67,8 +67,8 @@ assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
 async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
-if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
-while_stmt: 'while' test ':' suite ['else' ':' suite]
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
 ((except_clause ':' suite)+
@@ -91,6 +91,7 @@ testlist_safe: old_test [(',' old_test)+ [',']]
 old_test: or_test | old_lambdef
 old_lambdef: 'lambda' [varargslist] ':' old_test
 
+namedexpr_test: test [':=' test]
 test: or_test ['if' or_test 'else' test] | lambdef
 or_test: and_test ('or' and_test)*
 and_test: not_test ('and' not_test)*
@@ -111,8 +112,8 @@ atom: ('(' [yield_expr|testlist_gexp] ')' |
 '{' [dictsetmaker] '}' |
 '`' testlist1 '`' |
 NAME | NUMBER | STRING+ | '.' '.' '.')
-listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
-testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
 lambdef: 'lambda' [varargslist] ':' test
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
@@ -137,6 +138,7 @@ arglist: argument (',' argument)* [',']
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
 argument: ( test [comp_for] |
+ test ':=' test |
 test '=' test |
 '**' test |
 	 '*' test )
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
index 088c58bfa99c1..997fdf530f72b 100644
--- a/Lib/lib2to3/pgen2/grammar.py
+++ b/Lib/lib2to3/pgen2/grammar.py
@@ -202,6 +202,7 @@ def _make_deterministic(top):
 // DOUBLESLASH
 //= DOUBLESLASHEQUAL
 -> RARROW
+:= COLONEQUAL
 """
 
 opmap = {}
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 1a679554d2db4..5f6612f5b3068 100755
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -65,7 +65,8 @@
 AWAIT = 56
 ASYNC = 57
 ERRORTOKEN = 58
-N_TOKENS = 59
+COLONEQUAL = 59
+N_TOKENS = 60
 NT_OFFSET = 256
 #--end constants--
 
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 279d322971da9..94dd792805eaa 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -93,7 +93,7 @@ def _combinations(*l):
 r"~")
 
 Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'[:;.,`@]')
+Special = group(r'\r?\n', r':=', r'[:;.,`@]')
 Funny = group(Operator, Bracket, Special)
 
 PlainToken = group(Number, Funny, String, Name)
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 7ec881e01a9f8..753c846b7be7a 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -629,6 +629,21 @@ def test_multiline_str_literals(self):
 self.validate(s)
 
 
+class TestNamedAssignments(GrammarTest):
+
+ def test_named_assignment_if(self):
+ driver.parse_string("if f := x(): pass\n")
+
+ def test_named_assignment_while(self):
+ driver.parse_string("while f := x(): pass\n")
+
+ def test_named_assignment_generator(self):
+ driver.parse_string("any((lastNum := num) == 1 for num in [1, 2, 3])\n")
+
+ def test_named_assignment_listcomp(self):
+ driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n")
+
+
 def diff_texts(a, b, filename):
 a = a.splitlines()
 b = b.splitlines()
diff --git a/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst b/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst
new file mode 100644
index 0000000000000..e7b9dd648b407
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-18-19-38-27.bpo-36541.XI8mi1.rst
@@ -0,0 +1,2 @@
+lib2to3 now recognizes named assignment expressions (the walrus operator,
+``:=``)
