[Python-checkins] bpo-17560: Too small type for struct.pack/unpack in mutliprocessing.Connection (GH-10305)

Tue Nov 6 14:38:42 EST 2018

https://github.com/python/cpython/commit/bccacd19fa7b56dcf2fbfab15992b6b94ab6666b
commit: bccacd19fa7b56dcf2fbfab15992b6b94ab6666b
branch: master
author: Alexander Buchkovsky <olex.buchkovsky at gmail.com>
committer: Antoine Pitrou <pitrou at free.fr>
date: 2018年11月06日T20:38:34+01:00
summary:
bpo-17560: Too small type for struct.pack/unpack in mutliprocessing.Connection (GH-10305)
Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems.
files:
A Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst
M Lib/multiprocessing/connection.py

diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 1f3ea504fff4..c9f995e5fa7f 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -389,23 +389,33 @@ def _recv(self, size, read=_read):
 
 def _send_bytes(self, buf):
 n = len(buf)
- # For wire compatibility with 3.2 and lower
- header = struct.pack("!i", n)
- if n > 16384:
- # The payload is large so Nagle's algorithm won't be triggered
- # and we'd better avoid the cost of concatenation.
+ if n > 0x7fffffff:
+ pre_header = struct.pack("!i", -1)
+ header = struct.pack("!Q", n)
+ self._send(pre_header)
 self._send(header)
 self._send(buf)
 else:
- # Issue #20540: concatenate before sending, to avoid delays due
- # to Nagle's algorithm on a TCP socket.
- # Also note we want to avoid sending a 0-length buffer separately,
- # to avoid "broken pipe" errors if the other end closed the pipe.
- self._send(header + buf)
+ # For wire compatibility with 3.7 and lower
+ header = struct.pack("!i", n)
+ if n > 16384:
+ # The payload is large so Nagle's algorithm won't be triggered
+ # and we'd better avoid the cost of concatenation.
+ self._send(header)
+ self._send(buf)
+ else:
+ # Issue #20540: concatenate before sending, to avoid delays due
+ # to Nagle's algorithm on a TCP socket.
+ # Also note we want to avoid sending a 0-length buffer separately,
+ # to avoid "broken pipe" errors if the other end closed the pipe.
+ self._send(header + buf)
 
 def _recv_bytes(self, maxsize=None):
 buf = self._recv(4)
 size, = struct.unpack("!i", buf.getvalue())
+ if size == -1:
+ buf = self._recv(8)
+ size, = struct.unpack("!Q", buf.getvalue())
 if maxsize is not None and size > maxsize:
 return None
 return self._recv(size)
diff --git a/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst
new file mode 100644
index 000000000000..7cc9ed39007e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst
@@ -0,0 +1 @@
+Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems.
\ No newline at end of file
