[Python-checkins] r55078 - in sandbox/trunk/2to3: Grammar.txt pgen2/tokenize.py

Wed May 2 21:02:12 CEST 2007

Author: guido.van.rossum
Date: Wed May 2 21:02:07 2007
New Revision: 55078
Modified:
 sandbox/trunk/2to3/Grammar.txt
 sandbox/trunk/2to3/pgen2/tokenize.py
Log:
Hack the grammar to support nonlocal.
Add b"..." support to the tokenizer.
Modified: sandbox/trunk/2to3/Grammar.txt
==============================================================================

--- sandbox/trunk/2to3/Grammar.txt	(original)
+++ sandbox/trunk/2to3/Grammar.txt	Wed May 2 21:02:07 2007
@@ -76,7 +76,7 @@
 import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
-global_stmt: 'global' NAME (',' NAME)*
+global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
Modified: sandbox/trunk/2to3/pgen2/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/tokenize.py	(original)
+++ sandbox/trunk/2to3/pgen2/tokenize.py	Wed May 2 21:02:07 2007
@@ -87,9 +87,9 @@
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.
-ContStr = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
 group("'", r'\\\r?\n'),
- r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+ r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
 group('"', r'\\\r?\n'))
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
@@ -100,13 +100,21 @@
 "'''": single3prog, '"""': double3prog,
 "r'''": single3prog, 'r"""': double3prog,
 "u'''": single3prog, 'u"""': double3prog,
+ "b'''": single3prog, 'b"""': double3prog,
 "ur'''": single3prog, 'ur"""': double3prog,
+ "br'''": single3prog, 'br"""': double3prog,
 "R'''": single3prog, 'R"""': double3prog,
 "U'''": single3prog, 'U"""': double3prog,
+ "B'''": single3prog, 'B"""': double3prog,
 "uR'''": single3prog, 'uR"""': double3prog,
 "Ur'''": single3prog, 'Ur"""': double3prog,
 "UR'''": single3prog, 'UR"""': double3prog,
- 'r': None, 'R': None, 'u': None, 'U': None}
+ "bR'''": single3prog, 'bR"""': double3prog,
+ "Br'''": single3prog, 'Br"""': double3prog,
+ "BR'''": single3prog, 'BR"""': double3prog,
+ 'r': None, 'R': None,
+ 'u': None, 'U': None,
+ 'b': None, 'B': None}
 
 triple_quoted = {}
 for t in ("'''", '"""',
