[Python-checkins] cpython (3.4): Issue #20387: Restore retention of indentation during untokenize.

Sun Jun 28 17:17:50 CEST 2015

https://hg.python.org/cpython/rev/ff47efeeed48
changeset: 96709:ff47efeeed48
branch: 3.4
user: Dingyuan Wang <abcdoyle888 at gmail.com>
date: Mon Jun 22 10:01:12 2015 +0800
summary:
 Issue #20387: Restore retention of indentation during untokenize.
files:
 Lib/tokenize.py | 17 +++++++++++++++++
 1 files changed, 17 insertions(+), 0 deletions(-)

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -244,6 +244,8 @@
 
 def untokenize(self, iterable):
 it = iter(iterable)
+ indents = []
+ startline = False
 for t in it:
 if len(t) == 2:
 self.compat(t, it)
@@ -254,6 +256,21 @@
 continue
 if tok_type == ENDMARKER:
 break
+ if tok_type == INDENT:
+ indents.append(token)
+ continue
+ elif tok_type == DEDENT:
+ indents.pop()
+ self.prev_row, self.prev_col = end
+ continue
+ elif tok_type in (NEWLINE, NL):
+ startline = True
+ elif startline and indents:
+ indent = indents[-1]
+ if start[1] >= len(indent):
+ self.tokens.append(indent)
+ self.prev_col = len(indent)
+ startline = False
 self.add_whitespace(start)
 self.tokens.append(token)
 self.prev_row, self.prev_col = end
-- 
Repository URL: https://hg.python.org/cpython
