[Python-checkins] r70999 - in sandbox/trunk/2to3/lib2to3: main.py refactor.py

Thu Apr 2 00:36:47 CEST 2009

Author: benjamin.peterson
Date: Thu Apr 2 00:36:47 2009
New Revision: 70999
Log:
add very alpha support to 2to3 for running concurrently with multiprocessing
Modified:
 sandbox/trunk/2to3/lib2to3/main.py
 sandbox/trunk/2to3/lib2to3/refactor.py
Modified: sandbox/trunk/2to3/lib2to3/main.py
==============================================================================

--- sandbox/trunk/2to3/lib2to3/main.py	(original)
+++ sandbox/trunk/2to3/lib2to3/main.py	Thu Apr 2 00:36:47 2009
@@ -10,8 +10,12 @@
 
 from . import refactor
 
+HAVE_MULTIPROCESS = hasattr(refactor, "MultiprocessRefactoringTool")
 
-class StdoutRefactoringTool(refactor.RefactoringTool):
+base_refactor = getattr(refactor, "MultiprocessRefactoringTool",
+ refactor.RefactoringTool)
+
+class StdoutRefactoringTool(base_refactor):
 """
 Prints output to stdout.
 """
@@ -64,6 +68,9 @@
 help="Fix up doctests only")
 parser.add_option("-f", "--fix", action="append", default=[],
 help="Each FIX specifies a transformation; default: all")
+ if HAVE_MULTIPROCESS:
+ parser.add_option("-j", "--processes", action="store", default=1,
+ type="int", help="Run 2to3 concurrently")
 parser.add_option("-x", "--nofix", action="append", default=[],
 help="Prevent a fixer from being run.")
 parser.add_option("-l", "--list-fixes", action="store_true",
@@ -126,7 +133,8 @@
 if refactor_stdin:
 rt.refactor_stdin()
 else:
- rt.refactor(args, options.write, options.doctests_only)
+ rt.refactor(args, options.write, options.doctests_only,
+ options.processes)
 rt.summarize()
 
 # Return error status (0 if rt.errors is zero)
Modified: sandbox/trunk/2to3/lib2to3/refactor.py
==============================================================================
--- sandbox/trunk/2to3/lib2to3/refactor.py	(original)
+++ sandbox/trunk/2to3/lib2to3/refactor.py	Thu Apr 2 00:36:47 2009
@@ -506,6 +506,56 @@
 yield ""
 
 
+try:
+ import multiprocessing
+except ImportError:
+ pass
+else:
+ class MultiprocessRefactoringTool(RefactoringTool):
+
+ def __init__(self, *args, **kwargs):
+ super(MultiprocessRefactoringTool, self).__init__(*args, **kwargs)
+ self.queue = None
+
+ def refactor(self, items, write=False, doctests_only=False,
+ num_processes=1):
+ if num_processes == 1:
+ super(MultiprocessRefactoringTool, self).refactor(items, write,
+ doctests_only)
+ if self.queue is not None:
+ raise RuntimeError("already doing multiple processes")
+ self.queue = multiprocessing.JoinableQueue()
+ processes = [multiprocessing.Process(target=self._child)
+ for i in xrange(num_processes)]
+ try:
+ for p in processes:
+ p.start()
+ super(MultiprocessRefactoringTool, self).refactor(items, write,
+ doctests_only)
+ finally:
+ self.queue.join()
+ for i in xrange(num_processes):
+ self.queue.put(None)
+ for p in processes:
+ if p.is_alive():
+ p.join()
+ self.queue = None
+
+ def _child(self):
+ task = self.queue.get()
+ while task is not None:
+ args, kwargs = task
+ try:
+ super(MultiprocessRefactoringTool, self).refactor_file(
+ *args, **kwargs)
+ finally:
+ self.queue.task_done()
+ task = self.queue.get()
+
+ def refactor_file(self, *args, **kwargs):
+ self.queue.put((args, kwargs))
+
+
 def diff_texts(a, b, filename):
 """Return a unified diff of two strings."""
 a = a.splitlines()
