[Python-checkins] cpython: Port py_compile over to importlib

Sat Jan 26 14:50:07 CET 2013

http://hg.python.org/cpython/rev/d4eb02b6aac9
changeset: 81761:d4eb02b6aac9
user: Brett Cannon <brett at python.org>
date: Sat Jan 26 08:48:36 2013 -0500
summary:
 Port py_compile over to importlib
files:
 Lib/importlib/_bootstrap.py | 21 +-
 Lib/py_compile.py | 56 +-
 Misc/NEWS | 2 +
 Python/importlib.h | 8700 +++++++++++-----------
 4 files changed, 4396 insertions(+), 4383 deletions(-)

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -689,6 +689,15 @@
 raise ImportError("Non-code object in {!r}".format(bytecode_path),
 name=name, path=bytecode_path)
 
+def _code_to_bytecode(code, mtime=0, source_size=0):
+ """Compile a code object into bytecode for writing out to a byte-compiled
+ file."""
+ data = bytearray(_MAGIC_BYTES)
+ data.extend(_w_long(mtime))
+ data.extend(_w_long(source_size))
+ data.extend(marshal.dumps(code))
+ return data
+
 
 # Loaders #####################################################################
 
@@ -951,13 +960,13 @@
 raise ImportError("Failed to decode source file",
 name=fullname) from exc
 
- def source_to_code(self, data, path):
+ def source_to_code(self, data, path, *, _optimize=-1):
 """Return the code object compiled from source.
 
 The 'data' argument can be any object type that compile() supports.
 """
 return _call_with_frames_removed(compile, data, path, 'exec',
- dont_inherit=True)
+ dont_inherit=True, optimize=_optimize)
 
 def get_code(self, fullname):
 """Concrete implementation of InspectLoader.get_code.
@@ -1000,11 +1009,9 @@
 code_object = self.source_to_code(source_bytes, source_path)
 _verbose_message('code object from {}', source_path)
 if (not sys.dont_write_bytecode and bytecode_path is not None and
- source_mtime is not None):
- data = bytearray(_MAGIC_BYTES)
- data.extend(_w_long(source_mtime))
- data.extend(_w_long(len(source_bytes)))
- data.extend(marshal.dumps(code_object))
+ source_mtime is not None):
+ data = _code_to_bytecode(code_object, source_mtime,
+ len(source_bytes))
 try:
 self._cache_bytecode(source_path, bytecode_path, data)
 _verbose_message('wrote {!r}', bytecode_path)
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -3,17 +3,13 @@
 This module has intimate knowledge of the format of .pyc files.
 """
 
-import builtins
-import errno
 import imp
-import marshal
+import importlib._bootstrap
+import importlib.machinery
 import os
 import sys
-import tokenize
 import traceback
 
-MAGIC = imp.get_magic()
-
 __all__ = ["compile", "main", "PyCompileError"]
 
 
@@ -65,13 +61,6 @@
 return self.msg
 
 
-def wr_long(f, x):
- """Internal; write a 32-bit int to a file in little-endian order."""
- f.write(bytes([x & 0xff,
- (x >> 8) & 0xff,
- (x >> 16) & 0xff,
- (x >> 24) & 0xff]))
-
 def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
 """Byte-compile one Python source file to Python bytecode.
 
@@ -108,17 +97,16 @@
 byte-compile all installed files (or all files in selected
 directories).
 """
- with tokenize.open(file) as f:
- try:
- st = os.fstat(f.fileno())
- except AttributeError:
- st = os.stat(file)
- timestamp = int(st.st_mtime)
- size = st.st_size & 0xFFFFFFFF
- codestring = f.read()
+ if cfile is None:
+ if optimize >= 0:
+ cfile = imp.cache_from_source(file, debug_override=not optimize)
+ else:
+ cfile = imp.cache_from_source(file)
+ loader = importlib.machinery.SourceFileLoader('<py_compile>', file)
+ source_bytes = loader.get_data(file)
 try:
- codeobject = builtins.compile(codestring, dfile or file, 'exec',
- optimize=optimize)
+ code = loader.source_to_code(source_bytes, dfile or file,
+ _optimize=optimize)
 except Exception as err:
 py_exc = PyCompileError(err.__class__, err, dfile or file)
 if doraise:
@@ -126,26 +114,16 @@
 else:
 sys.stderr.write(py_exc.msg + '\n')
 return
- if cfile is None:
- if optimize >= 0:
- cfile = imp.cache_from_source(file, debug_override=not optimize)
- else:
- cfile = imp.cache_from_source(file)
 try:
 dirname = os.path.dirname(cfile)
 if dirname:
 os.makedirs(dirname)
- except OSError as error:
- if error.errno != errno.EEXIST:
- raise
- with open(cfile, 'wb') as fc:
- fc.write(b'0円0円0円0円')
- wr_long(fc, timestamp)
- wr_long(fc, size)
- marshal.dump(codeobject, fc)
- fc.flush()
- fc.seek(0, 0)
- fc.write(MAGIC)
+ except FileExistsError:
+ pass
+ source_stats = loader.path_stats(file)
+ bytecode = importlib._bootstrap._code_to_bytecode(code,
+ source_stats['mtime'], len(source_bytes))
+ loader._cache_bytecode(file, cfile, bytecode)
 return cfile
 
 def main(args=None):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -223,6 +223,8 @@
 Library
 -------
 
+Have py_compile use importlib as much as possible to avoid code duplication.
+
 - Issue #180022: Have site.addpackage() consider already known paths even when
 none are explicitly passed in. Bug report and fix by Kirill.
 
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]
-- 
Repository URL: http://hg.python.org/cpython
