[Python-checkins] r79067 - python/branches/py3k/Lib/tokenize.py

Thu Mar 18 23:34:15 CET 2010

Author: benjamin.peterson
Date: Thu Mar 18 23:34:15 2010
New Revision: 79067
Log:
fix for files with coding cookies and BOMs
Modified:
 python/branches/py3k/Lib/tokenize.py
Modified: python/branches/py3k/Lib/tokenize.py
==============================================================================

--- python/branches/py3k/Lib/tokenize.py	(original)
+++ python/branches/py3k/Lib/tokenize.py	Thu Mar 18 23:34:15 2010
@@ -333,9 +333,11 @@
 # This behaviour mimics the Python interpreter
 raise SyntaxError("unknown encoding: " + encoding)
 
- if bom_found and codec.name != 'utf-8':
- # This behaviour mimics the Python interpreter
- raise SyntaxError('encoding problem: utf-8')
+ if bom_found:
+ if codec.name != 'utf-8':
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError('encoding problem: utf-8')
+ encoding += '-sig'
 return encoding
 
 first = read_or_stop()
