[Python-checkins] cpython (2.7): Issue #15379: Fix passing of non-BMP characters as integers for the charmap

Sat Nov 17 21:17:15 CET 2012

http://hg.python.org/cpython/rev/c7ce91756472
changeset: 80478:c7ce91756472
branch: 2.7
parent: 80469:333fe4c4897a
user: Antoine Pitrou <solipsis at pitrou.net>
date: Sat Nov 17 21:14:58 2012 +0100
summary:
 Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings).
Patch by Serhiy Storchaka.
files:
 Lib/test/test_codeccallbacks.py | 2 +-
 Lib/test/test_codecs.py | 105 ++++++++++++++++++++
 Misc/NEWS | 3 +
 Objects/unicodeobject.c | 28 ++++-
 4 files changed, 135 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -717,7 +717,7 @@
 raise ValueError
 self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
 self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
- self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
+ self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: 0x110000})
 
 def test_encodehelper(self):
 # enhance coverage of:
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1534,6 +1534,10 @@
 (u"ab", 3)
 )
 
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab"
+ )
+
 self.assertEqual(
 codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
 (u"ab", 3)
@@ -1545,6 +1549,107 @@
 (u"", len(allbytes))
 )
 
+ def test_decode_with_int2str_map(self):
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: u'c'}),
+ (u"abc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'Aa', 1: u'Bb', 2: u'Cc'}),
+ (u"AaBbCc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'\U0010FFFF', 1: u'b', 2: u'c'}),
+ (u"\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: u''}),
+ (u"ab", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b'}
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b'}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b', 2: None}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b'}),
+ (u"ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b', 2: None}),
+ (u"ab", 3)
+ )
+
+ allbytes = bytes(range(256))
+ self.assertEqual(
+ codecs.charmap_decode(allbytes, "ignore", {}),
+ (u"", len(allbytes))
+ )
+
+ def test_decode_with_int2int_map(self):
+ a = ord(u'a')
+ b = ord(u'b')
+ c = ord(u'c')
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: c}),
+ (u"abc", 3)
+ )
+
+ # Issue #15379
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: 0x10FFFF, 1: b, 2: c}),
+ (u"\U0010FFFFbc", 3)
+ )
+
+ self.assertRaises(TypeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: 0x110000, 1: b, 2: c}
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: a, 1: b},
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: a, 1: b}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: a, 1: b}),
+ (u"ab", 3)
+ )
+
+
 class WithStmtTest(unittest.TestCase):
 def test_encodedfile(self):
 f = StringIO.StringIO("\xc3\xbc")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -9,6 +9,9 @@
 Core and Builtins
 -----------------
 
+- Issue #15379: Fix passing of non-BMP characters as integers for the charmap
+ decoder (already working as unicode strings). Patch by Serhiy Storchaka.
+
 - Issue #16453: Fix equality testing of dead weakref objects.
 
 - Issue #9535: Fix pending signals that have been received but not yet
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4127,12 +4127,36 @@
 /* Apply mapping */
 if (PyInt_Check(x)) {
 long value = PyInt_AS_LONG(x);
- if (value < 0 || value > 65535) {
+ if (value < 0 || value > 0x10FFFF) {
 PyErr_SetString(PyExc_TypeError,
- "character mapping must be in range(65536)");
+ "character mapping must be in range(0x110000)");
 Py_DECREF(x);
 goto onError;
 }
+
+#ifndef Py_UNICODE_WIDE
+ if (value > 0xFFFF) {
+ /* see the code for 1-n mapping below */
+ if (extrachars < 2) {
+ /* resize first */
+ Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
+ Py_ssize_t needed = 10 - extrachars;
+ extrachars += needed;
+ /* XXX overflow detection missing */
+ if (_PyUnicode_Resize(&v,
+ PyUnicode_GET_SIZE(v) + needed) < 0) {
+ Py_DECREF(x);
+ goto onError;
+ }
+ p = PyUnicode_AS_UNICODE(v) + oldpos;
+ }
+ value -= 0x10000;
+ *p++ = 0xD800 | (value >> 10);
+ *p++ = 0xDC00 | (value & 0x3FF);
+ extrachars -= 2;
+ }
+ else
+#endif
 *p++ = (Py_UNICODE)value;
 }
 else if (x == Py_None) {
-- 
Repository URL: http://hg.python.org/cpython
