[Python-checkins] r70772 - in python/trunk: Doc/library/email.header.rst Lib/email/generator.py Lib/email/test/test_email.py Lib/email/test/test_email_renamed.py

Tue Mar 31 00:42:17 CEST 2009

Author: barry.warsaw
Date: Tue Mar 31 00:42:17 2009
New Revision: 70772
Log:
A fix for issue 1974, inspired by the patch from Andi Albrecht (aalbrecht),
though with some changes by me. This patch should not be back ported or
forward ported. It's a bit too risky for 2.6 and 3.x does things fairly
differently.
Modified:
 python/trunk/Doc/library/email.header.rst
 python/trunk/Lib/email/generator.py
 python/trunk/Lib/email/test/test_email.py
 python/trunk/Lib/email/test/test_email_renamed.py
Modified: python/trunk/Doc/library/email.header.rst
==============================================================================

--- python/trunk/Doc/library/email.header.rst	(original)
+++ python/trunk/Doc/library/email.header.rst	Tue Mar 31 00:42:17 2009
@@ -74,7 +74,8 @@
 
 Optional *continuation_ws* must be :rfc:`2822`\ -compliant folding whitespace,
 and is usually either a space or a hard tab character. This character will be
- prepended to continuation lines.
+ prepended to continuation lines. *continuation_ws* defaults to a single
+ space character (" ").
 
 Optional *errors* is passed straight through to the :meth:`append` method.
 
Modified: python/trunk/Lib/email/generator.py
==============================================================================
--- python/trunk/Lib/email/generator.py	(original)
+++ python/trunk/Lib/email/generator.py	Tue Mar 31 00:42:17 2009
@@ -1,5 +1,4 @@
-# Copyright (C) 2001-2006 Python Software Foundation
-# Author: Barry Warsaw
+# Copyright (C) 2001-2009 Python Software Foundation
 # Contact: email-sig at python.org
 
 """Classes to generate plain text from a message object tree."""
@@ -156,10 +155,13 @@
 # be to not split the string and risk it being too long.
 print >> self._fp, v
 else:
- # Header's got lots of smarts, so use it.
+ # Header's got lots of smarts, so use it. Note that this is
+ # fundamentally broken though because we lose idempotency when
+ # the header string is continued with tabs. It will now be
+ # continued with spaces. This was reversedly broken before we
+ # fixed bug 1974. Either way, we lose.
 print >> self._fp, Header(
- v, maxlinelen=self._maxheaderlen,
- header_name=h, continuation_ws='\t').encode()
+ v, maxlinelen=self._maxheaderlen, header_name=h).encode()
 # A blank line always separates headers from body
 print >> self._fp
 
Modified: python/trunk/Lib/email/test/test_email.py
==============================================================================
--- python/trunk/Lib/email/test/test_email.py	(original)
+++ python/trunk/Lib/email/test/test_email.py	Tue Mar 31 00:42:17 2009
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2007 Python Software Foundation
+# Copyright (C) 2001-2009 Python Software Foundation
 # Contact: email-sig at python.org
 # email package unit tests
 
@@ -251,7 +251,16 @@
 msg = self._msgobj('msg_01.txt')
 fp = openfile('msg_01.txt')
 try:
- text = fp.read()
+ # BAW 30-Mar-2009 Evil be here. So, the generator is broken with
+ # respect to long line breaking. It's also not idempotent when a
+ # header from a parsed message is continued with tabs rather than
+ # spaces. Before we fixed bug 1974 it was reversedly broken,
+ # i.e. headers that were continued with spaces got continued with
+ # tabs. For Python 2.x there's really no good fix and in Python
+ # 3.x all this stuff is re-written to be right(er). Chris Withers
+ # convinced me that using space as the default continuation
+ # character is less bad for more applications.
+ text = fp.read().replace('\t', ' ')
 finally:
 fp.close()
 eq(text, msg.as_string())
@@ -554,8 +563,8 @@
 g.flatten(msg)
 eq(sfp.getvalue(), """\
 Subject: bug demonstration
-\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
-\tmore text
+ 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
+ more text
 
 test
 """)
@@ -655,7 +664,7 @@
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
-\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
+ spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
 
 ''')
 
@@ -671,7 +680,7 @@
 eq(sfp.getvalue(), """\
 From: test at dom.ain
 References: <0 at dom.ain> <1 at dom.ain> <2 at dom.ain> <3 at dom.ain> <4 at dom.ain>
-\t<5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain>
+ <5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain>
 
 Test""")
 
@@ -749,9 +758,9 @@
 msg['To'] = to
 eq(msg.as_string(0), '''\
 To: "Someone Test #A" <someone at eecs.umich.edu>, <someone at eecs.umich.edu>,
-\t"Someone Test #B" <someone at umich.edu>,
-\t"Someone Test #C" <someone at eecs.umich.edu>,
-\t"Someone Test #D" <someone at eecs.umich.edu>
+ "Someone Test #B" <someone at umich.edu>,
+ "Someone Test #C" <someone at eecs.umich.edu>,
+ "Someone Test #D" <someone at eecs.umich.edu>
 
 ''')
 
@@ -794,22 +803,22 @@
 \throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
 \tWed, 05 Mar 2003 18:10:18 -0700
 Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
-\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
-\tWed, 05 Mar 2003 18:10:18 -0700
+ hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
+ 2003年3月05日 18:10:18 -0700
 
 """)
 
 def test_string_headerinst_eq(self):
 h = '<15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")'
 msg = Message()
- msg['Received-1'] = Header(h, header_name='Received-1',
- continuation_ws='\t')
- msg['Received-2'] = h
- self.assertEqual(msg.as_string(), """\
-Received-1: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
-\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
-Received-2: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
+ msg['Received'] = Header(h, header_name='Received',
+ continuation_ws='\t')
+ msg['Received'] = h
+ self.ndiffAssertEqual(msg.as_string(), """\
+Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
 \t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
+Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
+ (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
 
 """)
 
@@ -823,7 +832,7 @@
 msg['Face-2'] = Header(t, header_name='Face-2')
 eq(msg.as_string(), """\
 Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
-\tlocQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
+ locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
 locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 
@@ -833,11 +842,11 @@
 eq = self.ndiffAssertEqual
 m = '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905);
-\tWed, 16 Oct 2002 07:41:11 -0700'''
+ 2002年10月16日 07:41:11 -0700'''
 msg = email.message_from_string(m)
 eq(msg.as_string(), '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with
-\tMicrosoft SMTPSVC(5.0.2195.4905); 2002年10月16日 07:41:11 -0700
+ Microsoft SMTPSVC(5.0.2195.4905); 2002年10月16日 07:41:11 -0700
 
 ''')
 
@@ -851,7 +860,7 @@
 msg['List'] = Header(h, header_name='List')
 eq(msg.as_string(), """\
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
-\t<mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
+ <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
 <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
 
@@ -2979,11 +2988,11 @@
 msg = self._msgobj('msg_01.txt')
 msg.set_param('title', 'This is even more ***fun*** isn\'t it!',
 charset='us-ascii', language='en')
- eq(msg.as_string(), """\
+ self.ndiffAssertEqual(msg.as_string(), """\
 Return-Path: <bbb at zzz.org>
 Delivered-To: bbb at zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684 at aaa.zzz.org>
@@ -2992,7 +3001,7 @@
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset=us-ascii;
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
@@ -3013,7 +3022,7 @@
 Return-Path: <bbb at zzz.org>
 Delivered-To: bbb at zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684 at aaa.zzz.org>
@@ -3022,7 +3031,7 @@
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset="us-ascii";
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
Modified: python/trunk/Lib/email/test/test_email_renamed.py
==============================================================================
--- python/trunk/Lib/email/test/test_email_renamed.py	(original)
+++ python/trunk/Lib/email/test/test_email_renamed.py	Tue Mar 31 00:42:17 2009
@@ -239,10 +239,19 @@
 msg = self._msgobj('msg_01.txt')
 fp = openfile('msg_01.txt')
 try:
- text = fp.read()
+ # BAW 30-Mar-2009 Evil be here. So, the generator is broken with
+ # respect to long line breaking. It's also not idempotent when a
+ # header from a parsed message is continued with tabs rather than
+ # spaces. Before we fixed bug 1974 it was reversedly broken,
+ # i.e. headers that were continued with spaces got continued with
+ # tabs. For Python 2.x there's really no good fix and in Python
+ # 3.x all this stuff is re-written to be right(er). Chris Withers
+ # convinced me that using space as the default continuation
+ # character is less bad for more applications.
+ text = fp.read().replace('\t', ' ')
 finally:
 fp.close()
- eq(text, msg.as_string())
+ self.ndiffAssertEqual(text, msg.as_string())
 fullrepr = str(msg)
 lines = fullrepr.split('\n')
 self.failUnless(lines[0].startswith('From '))
@@ -535,8 +544,8 @@
 g.flatten(msg)
 eq(sfp.getvalue(), """\
 Subject: bug demonstration
-\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
-\tmore text
+ 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789
+ more text
 
 test
 """)
@@ -636,7 +645,7 @@
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals";
-\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
+ spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"
 
 ''')
 
@@ -652,7 +661,7 @@
 eq(sfp.getvalue(), """\
 From: test at dom.ain
 References: <0 at dom.ain> <1 at dom.ain> <2 at dom.ain> <3 at dom.ain> <4 at dom.ain>
-\t<5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain>
+ <5 at dom.ain> <6 at dom.ain> <7 at dom.ain> <8 at dom.ain> <9 at dom.ain>
 
 Test""")
 
@@ -730,9 +739,9 @@
 msg['To'] = to
 eq(msg.as_string(0), '''\
 To: "Someone Test #A" <someone at eecs.umich.edu>, <someone at eecs.umich.edu>,
-\t"Someone Test #B" <someone at umich.edu>,
-\t"Someone Test #C" <someone at eecs.umich.edu>,
-\t"Someone Test #D" <someone at eecs.umich.edu>
+ "Someone Test #B" <someone at umich.edu>,
+ "Someone Test #C" <someone at eecs.umich.edu>,
+ "Someone Test #D" <someone at eecs.umich.edu>
 
 ''')
 
@@ -770,27 +779,27 @@
 msg = Message()
 msg['Received-1'] = Header(h, continuation_ws='\t')
 msg['Received-2'] = h
- self.assertEqual(msg.as_string(), """\
+ self.ndiffAssertEqual(msg.as_string(), """\
 Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
 \throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
 \tWed, 05 Mar 2003 18:10:18 -0700
 Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by
-\throthgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
-\tWed, 05 Mar 2003 18:10:18 -0700
+ hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP;
+ 2003年3月05日 18:10:18 -0700
 
 """)
 
 def test_string_headerinst_eq(self):
 h = '<15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de> (David Bremner\'s message of "Thu, 6 Mar 2003 13:58:21 +0100")'
 msg = Message()
- msg['Received-1'] = Header(h, header_name='Received-1',
- continuation_ws='\t')
- msg['Received-2'] = h
- self.assertEqual(msg.as_string(), """\
-Received-1: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
-\t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
-Received-2: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
+ msg['Received'] = Header(h, header_name='Received-1',
+ continuation_ws='\t')
+ msg['Received'] = h
+ self.ndiffAssertEqual(msg.as_string(), """\
+Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
 \t(David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
+Received: <15975.17901.207240.414604 at sgigritzmann1.mathematik.tu-muenchen.de>
+ (David Bremner's message of "Thu, 6 Mar 2003 13:58:21 +0100")
 
 """)
 
@@ -804,7 +813,7 @@
 msg['Face-2'] = Header(t, header_name='Face-2')
 eq(msg.as_string(), """\
 Face-1: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
-\tlocQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
+ locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 Face-2: iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9
 locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp
 
@@ -814,11 +823,11 @@
 eq = self.ndiffAssertEqual
 m = '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with Microsoft SMTPSVC(5.0.2195.4905);
-\tWed, 16 Oct 2002 07:41:11 -0700'''
+ 2002年10月16日 07:41:11 -0700'''
 msg = email.message_from_string(m)
 eq(msg.as_string(), '''\
 Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with
-\tMicrosoft SMTPSVC(5.0.2195.4905); 2002年10月16日 07:41:11 -0700
+ Microsoft SMTPSVC(5.0.2195.4905); 2002年10月16日 07:41:11 -0700
 
 ''')
 
@@ -830,9 +839,9 @@
 msg = Message()
 msg['List'] = h
 msg['List'] = Header(h, header_name='List')
- eq(msg.as_string(), """\
+ self.ndiffAssertEqual(msg.as_string(), """\
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
-\t<mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
+ <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
 List: List-Unsubscribe: <https://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,
 <mailto:spamassassin-talk-request at lists.sourceforge.net?subject=unsubscribe>
 
@@ -2974,11 +2983,11 @@
 msg = self._msgobj('msg_01.txt')
 msg.set_param('title', 'This is even more ***fun*** isn\'t it!',
 charset='us-ascii', language='en')
- eq(msg.as_string(), """\
+ self.ndiffAssertEqual(msg.as_string(), """\
 Return-Path: <bbb at zzz.org>
 Delivered-To: bbb at zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684 at aaa.zzz.org>
@@ -2987,7 +2996,7 @@
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset=us-ascii;
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
@@ -3008,7 +3017,7 @@
 Return-Path: <bbb at zzz.org>
 Delivered-To: bbb at zzz.org
 Received: by mail.zzz.org (Postfix, from userid 889)
-\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
+ id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT)
 MIME-Version: 1.0
 Content-Transfer-Encoding: 7bit
 Message-ID: <15090.61304.110929.45684 at aaa.zzz.org>
@@ -3017,7 +3026,7 @@
 Subject: This is a test message
 Date: Fri, 4 May 2001 14:05:44 -0400
 Content-Type: text/plain; charset="us-ascii";
-\ttitle*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
+ title*="us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21"
 
 
 Hi,
