[Python-checkins] cpython (2.7): #16152: fix tokenize to ignore whitespace at the end of the code when no

Sat Nov 3 16:51:39 CET 2012

http://hg.python.org/cpython/rev/eb7ea51e658e
changeset: 80190:eb7ea51e658e
branch: 2.7
parent: 80179:466cbfb3c0c6
user: Ezio Melotti <ezio.melotti at gmail.com>
date: Sat Nov 03 17:30:51 2012 +0200
summary:
 #16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder.
files:
 Lib/test/test_tokenize.py | 4 ++++
 Lib/tokenize.py | 4 +++-
 Misc/ACKS | 1 +
 Misc/NEWS | 3 +++
 4 files changed, 11 insertions(+), 1 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -550,6 +550,10 @@
 NAME 'pass' (3, 9) (3, 13)
 DEDENT '' (4, 0) (4, 0)
 DEDENT '' (4, 0) (4, 0)
+
+Pathological whitespace (http://bugs.python.org/issue16152)
+ >>> dump_tokens("@ ")
+ OP '@' (1, 0) (1, 1)
 """
 
 
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -95,7 +95,7 @@
 group("'", r'\\\r?\n'),
 r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
 group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 tokenprog, pseudoprog, single3prog, double3prog = map(
@@ -362,6 +362,8 @@
 if pseudomatch: # scan for tokens
 start, end = pseudomatch.span(1)
 spos, epos, pos = (lnum, start), (lnum, end), end
+ if start == end:
+ continue
 token, initial = line[start:end], line[start]
 
 if initial in numchars or \
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -64,6 +64,7 @@
 Ulf Bartelt
 Don Bashford
 Nick Bastin
+Ned Batchelder
 Jeff Bauer
 Mike Bayer
 Michael R Bax
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,9 @@
 Library
 -------
 
+- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
+ no newline is found. Patch by Ned Batchelder.
+
 - Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
 Patch by Todd Rovito.
 
-- 
Repository URL: http://hg.python.org/cpython
