[Python-checkins] cpython (2.7): Issue #9974: When untokenizing, use row info to insert backslash+newline.

Mon Feb 24 05:40:42 CET 2014

http://hg.python.org/cpython/rev/0f0e9b7d4f1d
changeset: 89352:0f0e9b7d4f1d
branch: 2.7
parent: 89347:a9464e900705
user: Terry Jan Reedy <tjreedy at udel.edu>
date: Sun Feb 23 23:32:59 2014 -0500
summary:
 Issue #9974: When untokenizing, use row info to insert backslash+newline.
Original patches by A. Kuchling and G. Rees (#12691).
files:
 Lib/test/test_tokenize.py | 16 +++++++++++++++-
 Lib/tokenize.py | 6 ++++++
 2 files changed, 21 insertions(+), 1 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -4,7 +4,7 @@
 >>> import glob, random, sys
 
 The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARKER is omitted for
 brevity.
 
 >>> dump_tokens("1 + 1")
@@ -618,6 +618,7 @@
 class UntokenizeTest(TestCase):
 
 def test_bad_input_order(self):
+ # raise if previous row
 u = Untokenizer()
 u.prev_row = 2
 u.prev_col = 2
@@ -625,8 +626,21 @@
 u.add_whitespace((1,3))
 self.assertEqual(cm.exception.args[0],
 'start (1,3) precedes previous end (2,2)')
+ # raise if previous column in row
 self.assertRaises(ValueError, u.add_whitespace, (2,1))
 
+ def test_backslash_continuation(self):
+ # The problem is that <whitespace>\<newline> leaves no token
+ u = Untokenizer()
+ u.prev_row = 1
+ u.prev_col = 1
+ u.tokens = []
+ u.add_whitespace((2, 0))
+ self.assertEqual(u.tokens, ['\\\n'])
+ u.prev_row = 2
+ u.add_whitespace((4, 4))
+ self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', ' '])
+
 def test_iter_compat(self):
 u = Untokenizer()
 token = (NAME, 'Hello')
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -188,6 +188,10 @@
 if row < self.prev_row or row == self.prev_row and col < self.prev_col:
 raise ValueError("start ({},{}) precedes previous end ({},{})"
 .format(row, col, self.prev_row, self.prev_col))
+ row_offset = row - self.prev_row
+ if row_offset:
+ self.tokens.append("\\\n" * row_offset)
+ self.prev_col = 0
 col_offset = col - self.prev_col
 if col_offset:
 self.tokens.append(" " * col_offset)
@@ -199,6 +203,8 @@
 self.compat(t, it)
 break
 tok_type, token, start, end, line = t
+ if tok_type == ENDMARKER:
+ break
 self.add_whitespace(start)
 self.tokens.append(token)
 self.prev_row, self.prev_col = end
-- 
Repository URL: http://hg.python.org/cpython
