[Python-checkins] cpython (2.7): Untokenize: An logically incorrect assert tested user input validity.

Mon Feb 17 22:50:20 CET 2014

http://hg.python.org/cpython/rev/c896d292080a
changeset: 89232:c896d292080a
branch: 2.7
parent: 89226:47394ceeaba1
user: Terry Jan Reedy <tjreedy at udel.edu>
date: Mon Feb 17 16:45:38 2014 -0500
summary:
 Untokenize: An logically incorrect assert tested user input validity.
Replace it with correct logic that raises ValueError for bad input.
Issues #8478 and #12691 reported the incorrect logic.
Add an Untokenize test case and an initial test method.
files:
 Lib/test/test_tokenize.py | 16 +++++++++++++++-
 Lib/tokenize.py | 4 +++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -559,9 +559,10 @@
 
 from test import test_support
 from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
- STRING, ENDMARKER, tok_name)
+ STRING, ENDMARKER, tok_name, Untokenizer)
 from StringIO import StringIO
 import os
+from unittest import TestCase
 
 def dump_tokens(s):
 """Print out the tokens in s in a table format.
@@ -614,12 +615,25 @@
 return untokenize(result)
 
 
+class UntokenizeTest(TestCase):
+
+ def test_bad_input_order(self):
+ u = Untokenizer()
+ u.prev_row = 2
+ u.prev_col = 2
+ with self.assertRaises(ValueError) as cm:
+ u.add_whitespace((1,3))
+ self.assertEqual(cm.exception.args[0],
+ 'start (1,3) precedes previous end (2,2)')
+ self.assertRaises(ValueError, u.add_whitespace, (2,1))
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 
 def test_main():
 from test import test_tokenize
 test_support.run_doctest(test_tokenize, True)
+ test_support.run_unittest(UntokenizeTest)
 
 if __name__ == "__main__":
 test_main()
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -184,7 +184,9 @@
 
 def add_whitespace(self, start):
 row, col = start
- assert row <= self.prev_row
+ if row < self.prev_row or row == self.prev_row and col < self.prev_col:
+ raise ValueError("start ({},{}) precedes previous end ({},{})"
+ .format(row, col, self.prev_row, self.prev_col))
 col_offset = col - self.prev_col
 if col_offset:
 self.tokens.append(" " * col_offset)
-- 
Repository URL: http://hg.python.org/cpython
