[Python-checkins] cpython (3.4): Issue #23215: Multibyte codecs with custom error handlers that ignores errors

Sat Feb 21 00:23:26 CET 2015

https://hg.python.org/cpython/rev/4dc8b7ed8973
changeset: 94712:4dc8b7ed8973
branch: 3.4
parent: 94709:36f5c36b7704
user: Serhiy Storchaka <storchaka at gmail.com>
date: Sat Feb 21 01:19:58 2015 +0200
summary:
 Issue #23215: Multibyte codecs with custom error handlers that ignores errors
consumed too much memory and raised SystemError or MemoryError.
Original patch by Aleksi Torhamo.
files:
 Lib/test/test_multibytecodec.py | 7 ++++++
 Misc/NEWS | 4 +++
 Modules/cjkcodecs/multibytecodec.c | 19 ++++++++++-------
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -44,6 +44,13 @@
 self.assertRaises(IndexError, dec,
 b'apple\x92ham\x93spam', 'test.cjktest')
 
+ def test_errorcallback_custom_ignore(self):
+ # Issue #23215: MemoryError with custom error handlers and multibyte codecs
+ data = 100 * "\udc00"
+ codecs.register_error("test.ignore", codecs.ignore_errors)
+ for enc in ALL_CJKENCODINGS:
+ self.assertEqual(data.encode(enc, "test.ignore"), b'')
+
 def test_codingspec(self):
 try:
 for enc in ALL_CJKENCODINGS:
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -13,6 +13,10 @@
 Library
 -------
 
+- Issue #23215: Multibyte codecs with custom error handlers that ignores errors
+ consumed too much memory and raised SystemError or MemoryError.
+ Original patch by Aleksi Torhamo.
+
 - Issue #5700: io.FileIO() called flush() after closing the file.
 flush() was not called in close() if closefd=False.
 
diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c
--- a/Modules/cjkcodecs/multibytecodec.c
+++ b/Modules/cjkcodecs/multibytecodec.c
@@ -182,8 +182,10 @@
 orgsize = PyBytes_GET_SIZE(buf->outobj);
 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
 
- if (orgsize > PY_SSIZE_T_MAX - incsize)
+ if (orgsize > PY_SSIZE_T_MAX - incsize) {
+ PyErr_NoMemory();
 return -1;
+ }
 
 if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
 return -1;
@@ -194,11 +196,11 @@
 
 return 0;
 }
-#define REQUIRE_ENCODEBUFFER(buf, s) { \
- if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
+#define REQUIRE_ENCODEBUFFER(buf, s) do { \
+ if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
 if (expand_encodebuffer(buf, s) == -1) \
 goto errorexit; \
-}
+} while(0)
 
 
 /**
@@ -332,10 +334,11 @@
 
 assert(PyBytes_Check(retstr));
 retstrsize = PyBytes_GET_SIZE(retstr);
- REQUIRE_ENCODEBUFFER(buf, retstrsize);
-
- memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
- buf->outbuf += retstrsize;
+ if (retstrsize > 0) {
+ REQUIRE_ENCODEBUFFER(buf, retstrsize);
+ memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
+ buf->outbuf += retstrsize;
+ }
 
 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
 if (newpos < 0 && !PyErr_Occurred())
-- 
Repository URL: https://hg.python.org/cpython
