changeset: 82206:0f827775f7b7 user: R David Murray date: Wed Feb 13 21:17:13 2013 -0500 files: Lib/email/feedparser.py Misc/NEWS description: #15220: simplify and speed up feedparser's line splitting. Original patch submitted by QNX, modified for clarity by me (mostly comments). QNX reports a 30% speed up in average email parsing time. diff -r 659ef9d360ae -r 0f827775f7b7 Lib/email/feedparser.py --- a/Lib/email/feedparser.py Wed Feb 13 15:25:21 2013 +0000 +++ b/Lib/email/feedparser.py Wed Feb 13 21:17:13 2013 -0500 @@ -98,24 +98,15 @@ """Push some new data into this object.""" # Handle any previous leftovers data, self._partial = self._partial + data, '' - # Crack into lines, but preserve the newlines on the end of each - parts = NLCRE_crack.split(data) - # The *ahem* interesting behaviour of re.split when supplied grouping - # parentheses is that the last element of the resulting list is the - # data after the final RE. In the case of a NL/CR terminated string, - # this is the empty string. - self._partial = parts.pop() - #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r: - # is there a \n to follow later? - if not self._partial and parts and parts[-1].endswith('\r'): - self._partial = parts.pop(-2)+parts.pop() - # parts is a list of strings, alternating between the line contents - # and the eol character(s). Gather up a list of lines after - # re-attaching the newlines. - lines = [] - for i in range(len(parts) // 2): - lines.append(parts[i*2] + parts[i*2+1]) - self.pushlines(lines) + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if parts and not parts[-1].endswith('\n'): + self._partial = parts.pop() + self.pushlines(parts) def pushlines(self, lines): # Reverse and insert at the front of the lines. diff -r 659ef9d360ae -r 0f827775f7b7 Misc/NEWS --- a/Misc/NEWS Wed Feb 13 15:25:21 2013 +0000 +++ b/Misc/NEWS Wed Feb 13 21:17:13 2013 -0500 @@ -253,6 +253,9 @@ Library ------- +- Issue #15220: email.feedparser's line splitting algorithm is now simpler and + faster. + - Issue #16743: Fix mmap overflow check on 32 bit Windows. - Issue #16996: webbrowser module now uses shutil.which() to find a