[Python-checkins] cpython (2.7): Issue #20387: Backport test from Python 3.4

Sun Jun 28 19:08:58 CEST 2015

https://hg.python.org/cpython/rev/524a0e755797
changeset: 96715:524a0e755797
branch: 2.7
parent: 96702:1f6c096ee772
user: Jason R. Coombs <jaraco at jaraco.com>
date: Sun Jun 28 13:03:26 2015 -0400
summary:
 Issue #20387: Backport test from Python 3.4
files:
 Lib/test/test_tokenize.py | 20 +++++++++++++++++++-
 1 files changed, 19 insertions(+), 1 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -559,7 +559,7 @@
 
 from test import test_support
 from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
- STRING, ENDMARKER, tok_name, Untokenizer)
+ STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
 from StringIO import StringIO
 import os
 from unittest import TestCase
@@ -650,12 +650,30 @@
 self.assertEqual(u.untokenize(iter([token])), 'Hello ')
 
 
+class TestRoundtrip(TestCase):
+ def roundtrip(self, code):
+ if isinstance(code, str):
+ code = code.encode('utf-8')
+ tokens = generate_tokens(StringIO(code).readline)
+ return untokenize(tokens).decode('utf-8')
+
+ def test_indentation_semantics_retained(self):
+ """
+ Ensure that although whitespace might be mutated in a roundtrip,
+ the semantic meaning of the indentation remains consistent.
+ """
+ code = "if False:\n\tx=3\n\tx=3\n"
+ codelines = self.roundtrip(code).split('\n')
+ self.assertEqual(codelines[1], codelines[2])
+
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 def test_main():
 from test import test_tokenize
 test_support.run_doctest(test_tokenize, True)
 test_support.run_unittest(UntokenizeTest)
+ test_support.run_unittest(TestRoundtrip)
 
 if __name__ == "__main__":
 test_main()
-- 
Repository URL: https://hg.python.org/cpython
