[Python-checkins] cpython: Issue #23611: Serializing more "lookupable" objects (such as unbound methods

Tue Mar 31 13:08:33 CEST 2015

https://hg.python.org/cpython/rev/ca12465418bd
changeset: 95320:ca12465418bd
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Mar 31 14:07:24 2015 +0300
summary:
 Issue #23611: Serializing more "lookupable" objects (such as unbound methods
or nested classes) now are supported with pickle protocols < 4.
files:
 Doc/whatsnew/3.5.rst | 7 +
 Lib/pickle.py | 33 ++++---
 Lib/test/pickletester.py | 19 +++-
 Misc/NEWS | 3 +
 Modules/_pickle.c | 120 ++++++++++++++++----------
 5 files changed, 115 insertions(+), 67 deletions(-)

diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -370,6 +370,13 @@
 * :class:`os.stat_result` now has a :attr:`~os.stat_result.st_file_attributes`
 attribute on Windows. (Contributed by Ben Hoyt in :issue:`21719`.)
 
+pickle
+------
+
+* Serializing more "lookupable" objects (such as unbound methods or nested
+ classes) now are supported with pickle protocols < 4.
+ (Contributed by Serhiy Storchaka in :issue:`23611`.)
+
 re
 --
 
diff --git a/Lib/pickle.py b/Lib/pickle.py
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -258,24 +258,20 @@
 
 # Tools used for pickling.
 
-def _getattribute(obj, name, allow_qualname=False):
- dotted_path = name.split(".")
- if not allow_qualname and len(dotted_path) > 1:
- raise AttributeError("Can't get qualified attribute {!r} on {!r}; " +
- "use protocols >= 4 to enable support"
- .format(name, obj))
- for subpath in dotted_path:
+def _getattribute(obj, name):
+ for subpath in name.split('.'):
 if subpath == '<locals>':
 raise AttributeError("Can't get local attribute {!r} on {!r}"
 .format(name, obj))
 try:
+ parent = obj
 obj = getattr(obj, subpath)
 except AttributeError:
 raise AttributeError("Can't get attribute {!r} on {!r}"
 .format(name, obj))
- return obj
+ return obj, parent
 
-def whichmodule(obj, name, allow_qualname=False):
+def whichmodule(obj, name):
 """Find the module an object belong to."""
 module_name = getattr(obj, '__module__', None)
 if module_name is not None:
@@ -286,7 +282,7 @@
 if module_name == '__main__' or module is None:
 continue
 try:
- if _getattribute(module, name, allow_qualname) is obj:
+ if _getattribute(module, name)[0] is obj:
 return module_name
 except AttributeError:
 pass
@@ -899,16 +895,16 @@
 write = self.write
 memo = self.memo
 
- if name is None and self.proto >= 4:
+ if name is None:
 name = getattr(obj, '__qualname__', None)
 if name is None:
 name = obj.__name__
 
- module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4)
+ module_name = whichmodule(obj, name)
 try:
 __import__(module_name, level=0)
 module = sys.modules[module_name]
- obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4)
+ obj2, parent = _getattribute(module, name)
 except (ImportError, KeyError, AttributeError):
 raise PicklingError(
 "Can't pickle %r: it's not found as %s.%s" %
@@ -930,11 +926,16 @@
 else:
 write(EXT4 + pack("<i", code))
 return
+ lastname = name.rpartition('.')[2]
+ if parent is module:
+ name = lastname
 # Non-ASCII identifiers are supported only with protocols >= 3.
 if self.proto >= 4:
 self.save(module_name)
 self.save(name)
 write(STACK_GLOBAL)
+ elif parent is not module:
+ self.save_reduce(getattr, (parent, lastname))
 elif self.proto >= 3:
 write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
 bytes(name, "utf-8") + b'\n')
@@ -1373,8 +1374,10 @@
 elif module in _compat_pickle.IMPORT_MAPPING:
 module = _compat_pickle.IMPORT_MAPPING[module]
 __import__(module, level=0)
- return _getattribute(sys.modules[module], name,
- allow_qualname=self.proto >= 4)
+ if self.proto >= 4:
+ return _getattribute(sys.modules[module], name)[0]
+ else:
+ return getattr(sys.modules[module], name)
 
 def load_reduce(self):
 stack = self.stack
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1602,13 +1602,24 @@
 class B:
 class C:
 pass
-
- for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
 with self.subTest(proto=proto, obj=obj):
 unpickled = self.loads(self.dumps(obj, proto))
 self.assertIs(obj, unpickled)
 
+ def test_recursive_nested_names(self):
+ global Recursive
+ class Recursive:
+ pass
+ Recursive.mod = sys.modules[Recursive.__module__]
+ Recursive.__qualname__ = 'Recursive.mod.Recursive'
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ with self.subTest(proto=proto):
+ unpickled = self.loads(self.dumps(Recursive, proto))
+ self.assertIs(unpickled, Recursive)
+ del Recursive.mod # break reference loop
+
 def test_py_methods(self):
 global PyMethodsTest
 class PyMethodsTest:
@@ -1647,7 +1658,7 @@
 (PyMethodsTest.biscuits, PyMethodsTest),
 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
 )
- for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
 for method in py_methods:
 with self.subTest(proto=proto, method=method):
 unpickled = self.loads(self.dumps(method, proto))
@@ -1687,7 +1698,7 @@
 (Subclass.Nested("sweet").count, ("e",)),
 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
 )
- for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
 for method, args in c_methods:
 with self.subTest(proto=proto, method=method):
 unpickled = self.loads(self.dumps(method, proto))
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -13,6 +13,9 @@
 Library
 -------
 
+- Issue #23611: Serializing more "lookupable" objects (such as unbound methods
+ or nested classes) now are supported with pickle protocols < 4.
+
 - Issue #13583: sqlite3.Row now supports slice indexing.
 
 - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -152,6 +152,8 @@
 
 /* codecs.encode, used for saving bytes in older protocols */
 PyObject *codecs_encode;
+ /* builtins.getattr, used for saving nested names with protocol < 4 */
+ PyObject *getattr;
 } PickleState;
 
 /* Forward declaration of the _pickle module definition. */
@@ -188,16 +190,26 @@
 Py_CLEAR(st->name_mapping_3to2);
 Py_CLEAR(st->import_mapping_3to2);
 Py_CLEAR(st->codecs_encode);
+ Py_CLEAR(st->getattr);
 }
 
 /* Initialize the given pickle module state. */
 static int
 _Pickle_InitState(PickleState *st)
 {
+ PyObject *builtins;
 PyObject *copyreg = NULL;
 PyObject *compat_pickle = NULL;
 PyObject *codecs = NULL;
 
+ builtins = PyEval_GetBuiltins();
+ if (builtins == NULL)
+ goto error;
+ st->getattr = PyDict_GetItemString(builtins, "getattr");
+ if (st->getattr == NULL)
+ goto error;
+ Py_INCREF(st->getattr);
+
 copyreg = PyImport_ImportModule("copyreg");
 if (!copyreg)
 goto error;
@@ -1535,7 +1547,7 @@
 }
 
 static PyObject *
-get_dotted_path(PyObject *obj, PyObject *name, int allow_qualname) {
+get_dotted_path(PyObject *obj, PyObject *name) {
 _Py_static_string(PyId_dot, ".");
 _Py_static_string(PyId_locals, "<locals>");
 PyObject *dotted_path;
@@ -1546,20 +1558,6 @@
 return NULL;
 n = PyList_GET_SIZE(dotted_path);
 assert(n >= 1);
- if (!allow_qualname && n > 1) {
- if (obj == NULL)
- PyErr_Format(PyExc_AttributeError,
- "Can't pickle qualified object %R; "
- "use protocols >= 4 to enable support",
- name);
- else
- PyErr_Format(PyExc_AttributeError,
- "Can't pickle qualified attribute %R on %R; "
- "use protocols >= 4 to enable support",
- name, obj);
- Py_DECREF(dotted_path);
- return NULL;
- }
 for (i = 0; i < n; i++) {
 PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
 PyObject *result = PyUnicode_RichCompare(
@@ -1582,22 +1580,28 @@
 }
 
 static PyObject *
-get_deep_attribute(PyObject *obj, PyObject *names)
+get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent)
 {
 Py_ssize_t i, n;
+ PyObject *parent = NULL;
 
 assert(PyList_CheckExact(names));
 Py_INCREF(obj);
 n = PyList_GET_SIZE(names);
 for (i = 0; i < n; i++) {
 PyObject *name = PyList_GET_ITEM(names, i);
- PyObject *tmp;
- tmp = PyObject_GetAttr(obj, name);
- Py_DECREF(obj);
- if (tmp == NULL)
+ Py_XDECREF(parent);
+ parent = obj;
+ obj = PyObject_GetAttr(parent, name);
+ if (obj == NULL) {
+ Py_DECREF(parent);
 return NULL;
- obj = tmp;
- }
+ }
+ }
+ if (pparent != NULL)
+ *pparent = parent;
+ else
+ Py_XDECREF(parent);
 return obj;
 }
 
@@ -1617,18 +1621,22 @@
 {
 PyObject *dotted_path, *attr;
 
- dotted_path = get_dotted_path(obj, name, allow_qualname);
- if (dotted_path == NULL)
- return NULL;
- attr = get_deep_attribute(obj, dotted_path);
- Py_DECREF(dotted_path);
+ if (allow_qualname) {
+ dotted_path = get_dotted_path(obj, name);
+ if (dotted_path == NULL)
+ return NULL;
+ attr = get_deep_attribute(obj, dotted_path, NULL);
+ Py_DECREF(dotted_path);
+ }
+ else
+ attr = PyObject_GetAttr(obj, name);
 if (attr == NULL)
 reformat_attribute_error(obj, name);
 return attr;
 }
 
 static PyObject *
-whichmodule(PyObject *global, PyObject *global_name, int allow_qualname)
+whichmodule(PyObject *global, PyObject *dotted_path)
 {
 PyObject *module_name;
 PyObject *modules_dict;
@@ -1637,7 +1645,6 @@
 _Py_IDENTIFIER(__module__);
 _Py_IDENTIFIER(modules);
 _Py_IDENTIFIER(__main__);
- PyObject *dotted_path;
 
 module_name = _PyObject_GetAttrId(global, &PyId___module__);
 
@@ -1663,10 +1670,6 @@
 return NULL;
 }
 
- dotted_path = get_dotted_path(NULL, global_name, allow_qualname);
- if (dotted_path == NULL)
- return NULL;
-
 i = 0;
 while (PyDict_Next(modules_dict, &i, &module_name, &module)) {
 PyObject *candidate;
@@ -1676,19 +1679,16 @@
 if (module == Py_None)
 continue;
 
- candidate = get_deep_attribute(module, dotted_path);
+ candidate = get_deep_attribute(module, dotted_path, NULL);
 if (candidate == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
- Py_DECREF(dotted_path);
+ if (!PyErr_ExceptionMatches(PyExc_AttributeError))
 return NULL;
- }
 PyErr_Clear();
 continue;
 }
 
 if (candidate == global) {
 Py_INCREF(module_name);
- Py_DECREF(dotted_path);
 Py_DECREF(candidate);
 return module_name;
 }
@@ -1698,7 +1698,6 @@
 /* If no module is found, use __main__. */
 module_name = _PyUnicode_FromId(&PyId___main__);
 Py_INCREF(module_name);
- Py_DECREF(dotted_path);
 return module_name;
 }
 
@@ -3105,6 +3104,9 @@
 PyObject *global_name = NULL;
 PyObject *module_name = NULL;
 PyObject *module = NULL;
+ PyObject *parent = NULL;
+ PyObject *dotted_path = NULL;
+ PyObject *lastname = NULL;
 PyObject *cls;
 PickleState *st = _Pickle_GetGlobalState();
 int status = 0;
@@ -3118,13 +3120,11 @@
 global_name = name;
 }
 else {
- if (self->proto >= 4) {
- global_name = _PyObject_GetAttrId(obj, &PyId___qualname__);
- if (global_name == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_AttributeError))
- goto error;
- PyErr_Clear();
- }
+ global_name = _PyObject_GetAttrId(obj, &PyId___qualname__);
+ if (global_name == NULL) {
+ if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+ goto error;
+ PyErr_Clear();
 }
 if (global_name == NULL) {
 global_name = _PyObject_GetAttrId(obj, &PyId___name__);
@@ -3133,7 +3133,10 @@
 }
 }
 
- module_name = whichmodule(obj, global_name, self->proto >= 4);
+ dotted_path = get_dotted_path(module, global_name);
+ if (dotted_path == NULL)
+ goto error;
+ module_name = whichmodule(obj, dotted_path);
 if (module_name == NULL)
 goto error;
 
@@ -3152,7 +3155,10 @@
 obj, module_name);
 goto error;
 }
- cls = getattribute(module, global_name, self->proto >= 4);
+ lastname = PyList_GET_ITEM(dotted_path, PyList_GET_SIZE(dotted_path)-1);
+ Py_INCREF(lastname);
+ cls = get_deep_attribute(module, dotted_path, &parent);
+ Py_CLEAR(dotted_path);
 if (cls == NULL) {
 PyErr_Format(st->PicklingError,
 "Can't pickle %R: attribute lookup %S on %S failed",
@@ -3239,6 +3245,11 @@
 }
 else {
 gen_global:
+ if (parent == module) {
+ Py_INCREF(lastname);
+ Py_DECREF(global_name);
+ global_name = lastname;
+ }
 if (self->proto >= 4) {
 const char stack_global_op = STACK_GLOBAL;
 
@@ -3250,6 +3261,15 @@
 if (_Pickler_Write(self, &stack_global_op, 1) < 0)
 goto error;
 }
+ else if (parent != module) {
+ PickleState *st = _Pickle_GetGlobalState();
+ PyObject *reduce_value = Py_BuildValue("(O(OO))",
+ st->getattr, parent, lastname);
+ status = save_reduce(self, reduce_value, NULL);
+ Py_DECREF(reduce_value);
+ if (status < 0)
+ goto error;
+ }
 else {
 /* Generate a normal global opcode if we are using a pickle
 protocol < 4, or if the object is not registered in the
@@ -3328,6 +3348,9 @@
 Py_XDECREF(module_name);
 Py_XDECREF(global_name);
 Py_XDECREF(module);
+ Py_XDECREF(parent);
+ Py_XDECREF(dotted_path);
+ Py_XDECREF(lastname);
 
 return status;
 }
@@ -7150,6 +7173,7 @@
 Py_VISIT(st->name_mapping_3to2);
 Py_VISIT(st->import_mapping_3to2);
 Py_VISIT(st->codecs_encode);
+ Py_VISIT(st->getattr);
 return 0;
 }
 
-- 
Repository URL: https://hg.python.org/cpython
