[Python-checkins] bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)

Fri Nov 23 12:41:59 EST 2018

https://github.com/python/cpython/commit/903a3e8d67b61594c0fa17fb201769ca924b38f8
commit: 903a3e8d67b61594c0fa17fb201769ca924b38f8
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: GitHub <noreply at github.com>
date: 2018年11月23日T09:41:54-08:00
summary:
bpo-29877: compileall: import ProcessPoolExecutor only when needed (GH-4856)
Importing ProcessPoolExecutor may hang or cause an error when the import
accesses urandom on a low resource platform
https://bugs.python.org/issue29877
(cherry picked from commit 1d817e4c8259f49602eefe9729743f6d9d748e8d)
Co-authored-by: Dustin Spicuzza <dustin at virtualroadside.com>
files:
A Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
M Lib/compileall.py
M Lib/test/test_compileall.py

diff --git a/Lib/compileall.py b/Lib/compileall.py
index 72592126d74c..40b148d2017e 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -16,10 +16,6 @@
 import py_compile
 import struct
 
-try:
- from concurrent.futures import ProcessPoolExecutor
-except ImportError:
- ProcessPoolExecutor = None
 from functools import partial
 
 __all__ = ["compile_dir","compile_file","compile_path"]
@@ -70,9 +66,17 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
 workers: maximum number of parallel workers
 invalidation_mode: how the up-to-dateness of the pyc will be checked
 """
- if workers is not None and workers < 0:
- raise ValueError('workers must be greater or equal to 0')
-
+ ProcessPoolExecutor = None
+ if workers is not None:
+ if workers < 0:
+ raise ValueError('workers must be greater or equal to 0')
+ elif workers != 1:
+ try:
+ # Only import when needed, as low resource platforms may
+ # fail to import it
+ from concurrent.futures import ProcessPoolExecutor
+ except ImportError:
+ workers = 1
 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
 ddir=ddir)
 success = True
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 38d7b996c7c2..2995e08aa8b5 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -161,7 +161,7 @@ def test_compile_dir_pathlike(self):
 self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)')
 self.assertTrue(os.path.isfile(self.bc_path))
 
- @mock.patch('compileall.ProcessPoolExecutor')
+ @mock.patch('concurrent.futures.ProcessPoolExecutor')
 def test_compile_pool_called(self, pool_mock):
 compileall.compile_dir(self.directory, quiet=True, workers=5)
 self.assertTrue(pool_mock.called)
@@ -171,19 +171,19 @@ def test_compile_workers_non_positive(self):
 "workers must be greater or equal to 0"):
 compileall.compile_dir(self.directory, workers=-1)
 
- @mock.patch('compileall.ProcessPoolExecutor')
+ @mock.patch('concurrent.futures.ProcessPoolExecutor')
 def test_compile_workers_cpu_count(self, pool_mock):
 compileall.compile_dir(self.directory, quiet=True, workers=0)
 self.assertEqual(pool_mock.call_args[1]['max_workers'], None)
 
- @mock.patch('compileall.ProcessPoolExecutor')
+ @mock.patch('concurrent.futures.ProcessPoolExecutor')
 @mock.patch('compileall.compile_file')
 def test_compile_one_worker(self, compile_file_mock, pool_mock):
 compileall.compile_dir(self.directory, quiet=True)
 self.assertFalse(pool_mock.called)
 self.assertTrue(compile_file_mock.called)
 
- @mock.patch('compileall.ProcessPoolExecutor', new=None)
+ @mock.patch('concurrent.futures.ProcessPoolExecutor', new=None)
 @mock.patch('compileall.compile_file')
 def test_compile_missing_multiprocessing(self, compile_file_mock):
 compileall.compile_dir(self.directory, quiet=True, workers=5)
diff --git a/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
new file mode 100644
index 000000000000..cc09533b7124
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-12-16-11-40-52.bpo-29877.SfWhmz.rst
@@ -0,0 +1,2 @@
+compileall: import ProcessPoolExecutor only when needed, preventing hangs on
+low resource platforms
