[Python-checkins] r61454 - in python/trunk: Lib/test/test_tokenize.py Lib/tokenize.py Misc/NEWS

Mon Mar 17 20:43:40 CET 2008

Author: eric.smith
Date: Mon Mar 17 20:43:40 2008
New Revision: 61454
Modified:
 python/trunk/Lib/test/test_tokenize.py
 python/trunk/Lib/tokenize.py
 python/trunk/Misc/NEWS
Log:
Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.
Modified: python/trunk/Lib/test/test_tokenize.py
==============================================================================

--- python/trunk/Lib/test/test_tokenize.py	(original)
+++ python/trunk/Lib/test/test_tokenize.py	Mon Mar 17 20:43:40 2008
@@ -4,7 +4,7 @@
 >>> import glob, random, sys
 
 The tests can be really simple. Given a small fragment of source
-code, print out a table with thokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARK is omitted for
 brevity.
 
 >>> dump_tokens("1 + 1")
@@ -106,7 +106,7 @@
 ... "else: print 'Loaded'\\n")
 True
 
-Balancing contunuation
+Balancing continuation
 
 >>> roundtrip("a = (3,4, \\n"
 ... "5,6)\\n"
@@ -126,6 +126,14 @@
 NUMBER '0xff' (1, 0) (1, 4)
 OP '<=' (1, 5) (1, 7)
 NUMBER '255' (1, 8) (1, 11)
+ >>> dump_tokens("0b10 <= 255")
+ NUMBER '0b10' (1, 0) (1, 4)
+ OP '<=' (1, 5) (1, 7)
+ NUMBER '255' (1, 8) (1, 11)
+ >>> dump_tokens("0o123 <= 0123")
+ NUMBER '0o123' (1, 0) (1, 5)
+ OP '<=' (1, 6) (1, 8)
+ NUMBER '0123' (1, 9) (1, 13)
 >>> dump_tokens("01234567 > ~0x15")
 NUMBER '01234567' (1, 0) (1, 8)
 OP '>' (1, 9) (1, 10)
Modified: python/trunk/Lib/tokenize.py
==============================================================================
--- python/trunk/Lib/tokenize.py	(original)
+++ python/trunk/Lib/tokenize.py	Mon Mar 17 20:43:40 2008
@@ -51,9 +51,10 @@
 Name = r'[a-zA-Z_]\w*'
 
 Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'0[0-7]*[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
 Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?\d+'
 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
 Expfloat = r'\d+' + Exponent
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS	(original)
+++ python/trunk/Misc/NEWS	Mon Mar 17 20:43:40 2008
@@ -12,6 +12,10 @@
 Core and builtins
 -----------------
 
+- PEP 3127: octal literals now start with "0o". Old-style octal literals
+ are still valid. There are binary literals with a prefix of "0b".
+ This also affects int(x, 0).
+
 - Issue #1779871: Gnu gcc can now build Python on OS X because the
 flags -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd are no
 longer passed.
