[Python-checkins] r74657 - in python/branches/release26-maint: Lib/httplib.py Misc/NEWS

Fri Sep 4 18:51:16 CEST 2009

Author: chris.withers
Date: Fri Sep 4 18:51:16 2009
New Revision: 74657
Log:
Fixes issue #6838: use a list to accumulate the value instead of repeatedly concatenating strings.
Modified:
 python/branches/release26-maint/Lib/httplib.py
 python/branches/release26-maint/Misc/NEWS
Modified: python/branches/release26-maint/Lib/httplib.py
==============================================================================

--- python/branches/release26-maint/Lib/httplib.py	(original)
+++ python/branches/release26-maint/Lib/httplib.py	Fri Sep 4 18:51:16 2009
@@ -544,10 +544,7 @@
 def _read_chunked(self, amt):
 assert self.chunked != _UNKNOWN
 chunk_left = self.chunk_left
- value = ''
-
- # XXX This accumulates chunks by repeated string concatenation,
- # which is not efficient as the number or size of chunks gets big.
+ value = []
 while True:
 if chunk_left is None:
 line = self.fp.readline()
@@ -560,22 +557,22 @@
 # close the connection as protocol synchronisation is
 # probably lost
 self.close()
- raise IncompleteRead(value)
+ raise IncompleteRead(''.join(value))
 if chunk_left == 0:
 break
 if amt is None:
- value += self._safe_read(chunk_left)
+ value.append(self._safe_read(chunk_left))
 elif amt < chunk_left:
- value += self._safe_read(amt)
+ value.append(self._safe_read(amt))
 self.chunk_left = chunk_left - amt
- return value
+ return ''.join(value)
 elif amt == chunk_left:
- value += self._safe_read(amt)
+ value.append(self._safe_read(amt))
 self._safe_read(2) # toss the CRLF at the end of the chunk
 self.chunk_left = None
- return value
+ return ''.join(value)
 else:
- value += self._safe_read(chunk_left)
+ value.append(self._safe_read(chunk_left))
 amt -= chunk_left
 
 # we read the whole chunk, get another
@@ -596,7 +593,7 @@
 # we read everything; close the "file"
 self.close()
 
- return value
+ return ''.join(value)
 
 def _safe_read(self, amt):
 """Read the number of bytes requested, compensating for partial reads.
Modified: python/branches/release26-maint/Misc/NEWS
==============================================================================
--- python/branches/release26-maint/Misc/NEWS	(original)
+++ python/branches/release26-maint/Misc/NEWS	Fri Sep 4 18:51:16 2009
@@ -72,6 +72,11 @@
 Library
 -------
 
+- Issue #6838: Use a list to accumulate the value instead of
+ repeatedly concatenating strings in httplib's
+ HTTPResponse._read_chunked providing a significant speed increase
+ when downloading large files servend with a Transfer-Encoding of 'chunked'.
+
 - Issue #6794: Fix Decimal.compare_total and Decimal.compare_total_mag: NaN
 payloads are now ordered by integer value rather than lexicographically.
 
