[Python-checkins] bpo-36867: Create the resource_tracker before launching SharedMemoryManagers (GH-13276)

Mon May 13 15:15:51 EDT 2019

https://github.com/python/cpython/commit/b1dfcad6f0d3a52c9ac31fb9763fc7962a84b27c
commit: b1dfcad6f0d3a52c9ac31fb9763fc7962a84b27c
branch: master
author: Pierre Glaser <pierreglaser at msn.com>
committer: Antoine Pitrou <antoine at python.org>
date: 2019年05月13日T21:15:32+02:00
summary:
bpo-36867: Create the resource_tracker before launching SharedMemoryManagers (GH-13276)
files:
A Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst
M Lib/multiprocessing/managers.py
M Lib/test/_test_multiprocessing.py

diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 2bad636855fe..514152298b09 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -21,6 +21,7 @@
 import array
 import queue
 import time
+import os
 from os import getpid
 
 from traceback import format_exc
@@ -1349,6 +1350,14 @@ class SharedMemoryManager(BaseManager):
 _Server = SharedMemoryServer
 
 def __init__(self, *args, **kwargs):
+ if os.name == "posix":
+ # bpo-36867: Ensure the resource_tracker is running before
+ # launching the manager process, so that concurrent
+ # shared_memory manipulation both in the manager and in the
+ # current process does not create two resource_tracker
+ # processes.
+ from . import resource_tracker
+ resource_tracker.ensure_running()
 BaseManager.__init__(self, *args, **kwargs)
 util.debug(f"{self.__class__.__name__} created by pid {getpid()}")
 
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index a50293c7616f..772c9638337a 100644
--- a/Lib/test/_test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -17,6 +17,7 @@
 import socket
 import random
 import logging
+import subprocess
 import struct
 import operator
 import pickle
@@ -3765,6 +3766,27 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
 
 smm.shutdown()
 
+ @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
+ def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self):
+ # bpo-36867: test that a SharedMemoryManager uses the
+ # same resource_tracker process as its parent.
+ cmd = '''if 1:
+ from multiprocessing.managers import SharedMemoryManager
+
+
+ smm = SharedMemoryManager()
+ smm.start()
+ sl = smm.ShareableList(range(10))
+ smm.shutdown()
+ '''
+ rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd)
+
+ # Before bpo-36867 was fixed, a SharedMemoryManager not using the same
+ # resource_tracker process as its parent would make the parent's
+ # tracker complain about sl being leaked even though smm.shutdown()
+ # properly released sl.
+ self.assertFalse(err)
+
 def test_shared_memory_SharedMemoryManager_basics(self):
 smm1 = multiprocessing.managers.SharedMemoryManager()
 with self.assertRaises(ValueError):
@@ -3904,8 +3926,6 @@ def test_shared_memory_ShareableList_pickling(self):
 sl.shm.close()
 
 def test_shared_memory_cleaned_after_process_termination(self):
- import subprocess
- from multiprocessing import shared_memory
 cmd = '''if 1:
 import os, time, sys
 from multiprocessing import shared_memory
@@ -3916,18 +3936,29 @@ def test_shared_memory_cleaned_after_process_termination(self):
 sys.stdout.flush()
 time.sleep(100)
 '''
- p = subprocess.Popen([sys.executable, '-E', '-c', cmd],
- stdout=subprocess.PIPE)
- name = p.stdout.readline().strip().decode()
+ with subprocess.Popen([sys.executable, '-E', '-c', cmd],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE) as p:
+ name = p.stdout.readline().strip().decode()
 
- # killing abruptly processes holding reference to a shared memory
- # segment should not leak the given memory segment.
- p.terminate()
- p.wait()
- time.sleep(1.0) # wait for the OS to collect the segment
+ # killing abruptly processes holding reference to a shared memory
+ # segment should not leak the given memory segment.
+ p.terminate()
+ p.wait()
+ time.sleep(1.0) # wait for the OS to collect the segment
 
- with self.assertRaises(FileNotFoundError):
- smm = shared_memory.SharedMemory(name, create=False)
+ # The shared memory file was deleted.
+ with self.assertRaises(FileNotFoundError):
+ smm = shared_memory.SharedMemory(name, create=False)
+
+ if os.name == 'posix':
+ # A warning was emitted by the subprocess' own
+ # resource_tracker (on Windows, shared memory segments
+ # are released automatically by the OS).
+ err = p.stderr.read().decode()
+ self.assertIn(
+ "resource_tracker: There appear to be 1 leaked "
+ "shared_memory objects to clean up at shutdown", err)
 
 #
 #
@@ -4560,7 +4591,7 @@ def run_in_child(cls):
 print(json.dumps(flags))
 
 def test_flags(self):
- import json, subprocess
+ import json
 # start child process using unusual flags
 prog = ('from test._test_multiprocessing import TestFlags; ' +
 'TestFlags.run_in_child()')
@@ -4866,7 +4897,6 @@ def test_resource_tracker(self):
 #
 # Check that killing process does not leak named semaphores
 #
- import subprocess
 cmd = '''if 1:
 import time, os, tempfile
 import multiprocessing as mp
diff --git a/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst b/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst
new file mode 100644
index 000000000000..ce925d0594bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst
@@ -0,0 +1 @@
+Fix a bug making a SharedMemoryManager instance and its parent process use two separate resource_tracker processes.
\ No newline at end of file
