[Python-checkins] cpython: try to use the same str object for all code filenames when compiling or

Fri May 27 16:07:47 CEST 2011

http://hg.python.org/cpython/rev/27359a4e0f8c
changeset: 70435:27359a4e0f8c
user: Benjamin Peterson <benjamin at python.org>
date: Fri May 27 09:08:01 2011 -0500
summary:
 try to use the same str object for all code filenames when compiling or unmarshalling (#12190)
This should reduce memory usage.
files:
 Lib/test/test_compile.py | 9 +++++++++
 Lib/test/test_marshal.py | 17 +++++++++++++++++
 Misc/NEWS | 3 +++
 Python/compile.c | 27 +++++++++------------------
 Python/marshal.c | 19 +++++++++++++++++++
 5 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -1,6 +1,7 @@
 import unittest
 import sys
 import _ast
+import types
 from test import support
 
 class TestSpecifics(unittest.TestCase):
@@ -433,6 +434,14 @@
 ast.body = [_ast.BoolOp()]
 self.assertRaises(TypeError, compile, ast, '<ast>', 'exec')
 
+ @support.cpython_only
+ def test_same_filename_used(self):
+ s = """def f(): pass\ndef g(): pass"""
+ c = compile(s, "myfile", "exec")
+ for obj in c.co_consts:
+ if isinstance(obj, types.CodeType):
+ self.assertIs(obj.co_filename, c.co_filename)
+
 
 def test_main():
 support.run_unittest(TestSpecifics)
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -5,6 +5,7 @@
 import sys
 import unittest
 import os
+import types
 
 class HelperMixin:
 def helper(self, sample, *extra):
@@ -113,6 +114,22 @@
 codes = (ExceptionTestCase.test_exceptions.__code__,) * count
 marshal.loads(marshal.dumps(codes))
 
+ def test_different_filenames(self):
+ co1 = compile("x", "f1", "exec")
+ co2 = compile("y", "f2", "exec")
+ co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
+ self.assertEqual(co1.co_filename, "f1")
+ self.assertEqual(co2.co_filename, "f2")
+
+ @support.cpython_only
+ def test_same_filename_used(self):
+ s = """def f(): pass\ndef g(): pass"""
+ co = compile(s, "myfile", "exec")
+ co = marshal.loads(marshal.dumps(co))
+ for obj in co.co_consts:
+ if isinstance(obj, types.CodeType):
+ self.assertIs(co.co_filename, obj.co_filename)
+
 class ContainerTestCase(unittest.TestCase, HelperMixin):
 d = {'astring': 'foo at bar.baz.spam',
 'afloat': 7283.43,
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #12190: Try to use the same filename object when compiling unmarshalling
+ a code objects in the same file.
+
 - Issue #12166: Move implementations of dir() specialized for various types into
 the __dir__() methods of those types.
 
diff --git a/Python/compile.c b/Python/compile.c
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -135,6 +135,7 @@
 
 struct compiler {
 const char *c_filename;
+ PyObject *c_filename_obj;
 struct symtable *c_st;
 PyFutureFeatures *c_future; /* pointer to module's __future__ */
 PyCompilerFlags *c_flags;
@@ -272,6 +273,9 @@
 if (!compiler_init(&c))
 return NULL;
 c.c_filename = filename;
+ c.c_filename_obj = PyUnicode_DecodeFSDefault(filename);
+ if (!c.c_filename_obj)
+ goto finally;
 c.c_arena = arena;
 c.c_future = PyFuture_FromAST(mod, filename);
 if (c.c_future == NULL)
@@ -324,6 +328,8 @@
 PySymtable_Free(c->c_st);
 if (c->c_future)
 PyObject_Free(c->c_future);
+ if (c->c_filename_obj)
+ Py_DECREF(c->c_filename_obj);
 Py_DECREF(c->c_stack);
 }
 
@@ -3361,7 +3367,7 @@
 static int
 compiler_error(struct compiler *c, const char *errstr)
 {
- PyObject *loc, *filename;
+ PyObject *loc;
 PyObject *u = NULL, *v = NULL;
 
 loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno);
@@ -3369,16 +3375,7 @@
 Py_INCREF(Py_None);
 loc = Py_None;
 }
- if (c->c_filename != NULL) {
- filename = PyUnicode_DecodeFSDefault(c->c_filename);
- if (!filename)
- goto exit;
- }
- else {
- Py_INCREF(Py_None);
- filename = Py_None;
- }
- u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno,
+ u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno,
 c->u->u_col_offset, loc);
 if (!u)
 goto exit;
@@ -3927,7 +3924,6 @@
 PyObject *consts = NULL;
 PyObject *names = NULL;
 PyObject *varnames = NULL;
- PyObject *filename = NULL;
 PyObject *name = NULL;
 PyObject *freevars = NULL;
 PyObject *cellvars = NULL;
@@ -3951,10 +3947,6 @@
 freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
 if (!freevars)
 goto error;
- filename = PyUnicode_DecodeFSDefault(c->c_filename);
- if (!filename)
- goto error;
-
 nlocals = PyDict_Size(c->u->u_varnames);
 flags = compute_code_flags(c);
 if (flags < 0)
@@ -3974,14 +3966,13 @@
 nlocals, stackdepth(c), flags,
 bytecode, consts, names, varnames,
 freevars, cellvars,
- filename, c->u->u_name,
+ c->c_filename_obj, c->u->u_name,
 c->u->u_firstlineno,
 a->a_lnotab);
 error:
 Py_XDECREF(consts);
 Py_XDECREF(names);
 Py_XDECREF(varnames);
- Py_XDECREF(filename);
 Py_XDECREF(name);
 Py_XDECREF(freevars);
 Py_XDECREF(cellvars);
diff --git a/Python/marshal.c b/Python/marshal.c
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -58,6 +58,7 @@
 int depth;
 /* If fp == NULL, the following are valid: */
 PyObject *str;
+ PyObject *current_filename;
 char *ptr;
 char *end;
 int version;
@@ -976,6 +977,18 @@
 filename = r_object(p);
 if (filename == NULL)
 goto code_error;
+ if (PyUnicode_CheckExact(filename)) {
+ if (p->current_filename != NULL) {
+ if (!PyUnicode_Compare(filename, p->current_filename)) {
+ Py_DECREF(filename);
+ Py_INCREF(p->current_filename);
+ filename = p->current_filename;
+ }
+ }
+ else {
+ p->current_filename = filename;
+ }
+ }
 name = r_object(p);
 if (name == NULL)
 goto code_error;
@@ -1037,6 +1050,7 @@
 RFILE rf;
 assert(fp);
 rf.fp = fp;
+ rf.current_filename = NULL;
 rf.end = rf.ptr = NULL;
 return r_short(&rf);
 }
@@ -1046,6 +1060,7 @@
 {
 RFILE rf;
 rf.fp = fp;
+ rf.current_filename = NULL;
 rf.ptr = rf.end = NULL;
 return r_long(&rf);
 }
@@ -1106,6 +1121,7 @@
 RFILE rf;
 PyObject *result;
 rf.fp = fp;
+ rf.current_filename = NULL;
 rf.depth = 0;
 rf.ptr = rf.end = NULL;
 result = r_object(&rf);
@@ -1118,6 +1134,7 @@
 RFILE rf;
 PyObject *result;
 rf.fp = NULL;
+ rf.current_filename = NULL;
 rf.ptr = str;
 rf.end = str + len;
 rf.depth = 0;
@@ -1214,6 +1231,7 @@
 if (data == NULL)
 return NULL;
 rf.fp = NULL;
+ rf.current_filename = NULL;
 if (PyBytes_Check(data)) {
 rf.ptr = PyBytes_AS_STRING(data);
 rf.end = rf.ptr + PyBytes_GET_SIZE(data);
@@ -1282,6 +1300,7 @@
 s = p.buf;
 n = p.len;
 rf.fp = NULL;
+ rf.current_filename = NULL;
 rf.ptr = s;
 rf.end = s + n;
 rf.depth = 0;
-- 
Repository URL: http://hg.python.org/cpython
