[Python-checkins] r74063 - in python/branches/py3k: Doc/library/multiprocessing.rst Lib/multiprocessing/process.py Lib/test/test_multiprocessing.py Misc/ACKS

Fri Jul 17 14:07:01 CEST 2009

Author: alexandre.vassalotti
Date: Fri Jul 17 14:07:01 2009
New Revision: 74063
Log:
Merged revisions 73694,73708,73738 via svnmerge from 
svn+ssh://pythondev@svn.python.org/python/trunk
........
 r73694 | jesse.noller | 2009年06月29日 14:24:26 -0400 (2009年6月29日) | 1 line
 
 Issue 5740: multiprocessing.connection.* authkey fixes
........
 r73708 | jesse.noller | 2009年06月30日 13:11:52 -0400 (2009年6月30日) | 1 line
 
 Resolves issues 5155, 5313, 5331 - bad file descriptor error with processes in processes
........
 r73738 | r.david.murray | 2009年06月30日 22:49:10 -0400 (2009年6月30日) | 2 lines
 
 Make punctuation prettier and break up run-on sentence.
........
Modified:
 python/branches/py3k/ (props changed)
 python/branches/py3k/Doc/library/multiprocessing.rst
 python/branches/py3k/Lib/multiprocessing/process.py
 python/branches/py3k/Lib/test/test_multiprocessing.py
 python/branches/py3k/Misc/ACKS
Modified: python/branches/py3k/Doc/library/multiprocessing.rst
==============================================================================

--- python/branches/py3k/Doc/library/multiprocessing.rst	(original)
+++ python/branches/py3k/Doc/library/multiprocessing.rst	Fri Jul 17 14:07:01 2009
@@ -1715,7 +1715,7 @@
 generally be omitted since it can usually be inferred from the format of
 *address*. (See :ref:`multiprocessing-address-formats`)
 
- If *authentication* is ``True`` or *authkey* is a string then digest
+ If *authenticate* is ``True`` or *authkey* is a string then digest
 authentication is used. The key used for authentication will be either
 *authkey* or ``current_process().authkey)`` if *authkey* is ``None``.
 If authentication fails then :exc:`AuthenticationError` is raised. See
@@ -1757,7 +1757,7 @@
 
 If *authkey* is ``None`` and *authenticate* is ``True`` then
 ``current_process().authkey`` is used as the authentication key. If
- *authkey* is ``None`` and *authentication* is ``False`` then no
+ *authkey* is ``None`` and *authenticate* is ``False`` then no
 authentication is done. If authentication fails then
 :exc:`AuthenticationError` is raised. See :ref:`multiprocessing-auth-keys`.
 
@@ -2099,6 +2099,38 @@
 for i in range(10):
 Process(target=f, args=(lock,)).start()
 
+Beware replacing sys.stdin with a "file like object"
+
+ :mod:`multiprocessing` originally unconditionally called::
+
+ os.close(sys.stdin.fileno())
+
+ in the :meth:`multiprocessing.Process._bootstrap` method --- this resulted
+ in issues with processes-in-processes. This has been changed to::
+
+ sys.stdin.close()
+ sys.stdin = open(os.devnull)
+
+ Which solves the fundamental issue of processes colliding with each other
+ resulting in a bad file descriptor error, but introduces a potential danger
+ to applications which replace :func:`sys.stdin` with a "file-like object"
+ with output buffering. This danger is that if multiple processes call
+ :func:`close()` on this file-like object, it could result in the same
+ data being flushed to the object multiple times, resulting in corruption.
+
+ If you write a file-like object and implement your own caching, you can
+ make it fork-safe by storing the pid whenever you append to the cache,
+ and discarding the cache when the pid changes. For example::
+
+ @property
+ def cache(self):
+ pid = os.getpid()
+ if pid != self._pid:
+ self._pid = pid
+ self._cache = []
+ return self._cache
+
+ For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331`
 
 Windows
 ~~~~~~~
Modified: python/branches/py3k/Lib/multiprocessing/process.py
==============================================================================
--- python/branches/py3k/Lib/multiprocessing/process.py	(original)
+++ python/branches/py3k/Lib/multiprocessing/process.py	Fri Jul 17 14:07:01 2009
@@ -221,7 +221,8 @@
 self._counter = itertools.count(1)
 if sys.stdin is not None:
 try:
- os.close(sys.stdin.fileno())
+ sys.stdin.close()
+ sys.stdin = open(os.devnull)
 except (OSError, ValueError):
 pass
 _current_process = self
Modified: python/branches/py3k/Lib/test/test_multiprocessing.py
==============================================================================
--- python/branches/py3k/Lib/test/test_multiprocessing.py	(original)
+++ python/branches/py3k/Lib/test/test_multiprocessing.py	Fri Jul 17 14:07:01 2009
@@ -8,6 +8,7 @@
 import threading
 import queue as pyqueue
 import time
+import io
 import sys
 import os
 import gc
@@ -1868,7 +1869,74 @@
 p.join()
 self.assertEqual(self.ns.test, 1)
 
-testcases_other = [OtherTest, TestInvalidHandle, TestInitializers]
+#
+# Issue 5155, 5313, 5331: Test process in processes
+# Verifies os.close(sys.stdin.fileno) vs. sys.stdin.close() behavior
+#
+
+def _ThisSubProcess(q):
+ try:
+ item = q.get(block=False)
+ except pyqueue.Empty:
+ pass
+
+def _TestProcess(q):
+ queue = multiprocessing.Queue()
+ subProc = multiprocessing.Process(target=_ThisSubProcess, args=(queue,))
+ subProc.start()
+ subProc.join()
+
+def _afunc(x):
+ return x*x
+
+def pool_in_process():
+ pool = multiprocessing.Pool(processes=4)
+ x = pool.map(_afunc, [1, 2, 3, 4, 5, 6, 7])
+
+class _file_like(object):
+ def __init__(self, delegate):
+ self._delegate = delegate
+ self._pid = None
+
+ @property
+ def cache(self):
+ pid = os.getpid()
+ # There are no race conditions since fork keeps only the running thread
+ if pid != self._pid:
+ self._pid = pid
+ self._cache = []
+ return self._cache
+
+ def write(self, data):
+ self.cache.append(data)
+
+ def flush(self):
+ self._delegate.write(''.join(self.cache))
+ self._cache = []
+
+class TestStdinBadfiledescriptor(unittest.TestCase):
+
+ def test_queue_in_process(self):
+ queue = multiprocessing.Queue()
+ proc = multiprocessing.Process(target=_TestProcess, args=(queue,))
+ proc.start()
+ proc.join()
+
+ def test_pool_in_process(self):
+ p = multiprocessing.Process(target=pool_in_process)
+ p.start()
+ p.join()
+
+ def test_flushing(self):
+ sio = io.StringIO()
+ flike = _file_like(sio)
+ flike.write('foo')
+ proc = multiprocessing.Process(target=lambda: flike.flush())
+ flike.flush()
+ assert sio.getvalue() == 'foo'
+
+testcases_other = [OtherTest, TestInvalidHandle, TestInitializers,
+ TestStdinBadfiledescriptor]
 
 #
 #
Modified: python/branches/py3k/Misc/ACKS
==============================================================================
--- python/branches/py3k/Misc/ACKS	(original)
+++ python/branches/py3k/Misc/ACKS	Fri Jul 17 14:07:01 2009
@@ -46,6 +46,7 @@
 Ulf Bartelt
 Nick Bastin
 Jeff Bauer
+Mike Bayer
 Michael R Bax
 Anthony Baxter
 Samuel L. Bayer
@@ -185,6 +186,7 @@
 Dean Draayer
 John DuBois
 Paul Dubois
+Graham Dumpleton
 Quinn Dunkan
 Robin Dunn
 Luke Dunstan
@@ -558,6 +560,7 @@
 Santiago Peresón
 Mark Perrego
 Trevor Perrin
+Gabriel de Perthuis
 Tim Peters
 Benjamin Peterson
 Chris Petrilli
