[Python-checkins] cpython (merge 3.4 -> 3.5): Issue #24802: Merge null termination fixes from 3.4 into 3.5

Fri Nov 6 22:33:10 EST 2015

https://hg.python.org/cpython/rev/95b9c07b27f7
changeset: 98993:95b9c07b27f7
branch: 3.5
parent: 98990:de79e483565c
parent: 98992:a75336ac40e0
user: Martin Panter <vadmium+py at gmail.com>
date: Sat Nov 07 02:56:11 2015 +0000
summary:
 Issue #24802: Merge null termination fixes from 3.4 into 3.5
files:
 Lib/test/test_compile.py | 21 ++++++++++++
 Lib/test/test_float.py | 38 +++++++++++++++++++++-
 Lib/test/test_int.py | 42 +++++++++++++++++++-----
 Misc/NEWS | 4 ++
 Objects/abstract.c | 22 +++++++++++-
 Objects/complexobject.c | 7 ----
 Objects/floatobject.c | 15 ++++++++
 Python/bltinmodule.c | 48 ++++++++++++++++++---------
 8 files changed, 161 insertions(+), 36 deletions(-)

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -542,6 +542,27 @@
 check_limit("a", "[0]")
 check_limit("a", "*a")
 
+ def test_null_terminated(self):
+ # The source code is null-terminated internally, but bytes-like
+ # objects are accepted, which could be not terminated.
+ # Exception changed from TypeError to ValueError in 3.5
+ with self.assertRaisesRegex(Exception, "cannot contain null"):
+ compile("123\x00", "<dummy>", "eval")
+ with self.assertRaisesRegex(Exception, "cannot contain null"):
+ compile(memoryview(b"123\x00"), "<dummy>", "eval")
+ code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")
+ self.assertEqual(eval(code), 23)
+ code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval")
+ self.assertEqual(eval(code), 23)
+ code = compile(memoryview(b"23ドル$")[1:-1], "<dummy>", "eval")
+ self.assertEqual(eval(code), 23)
+
+ # Also test when eval() and exec() do the compilation step
+ self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23)
+ namespace = dict()
+ exec(memoryview(b"ax = 123")[1:-1], namespace)
+ self.assertEqual(namespace['x'], 12)
+
 
 class TestStackSize(unittest.TestCase):
 # These tests check that the computed stack size for a code object
diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -31,7 +31,6 @@
 self.assertEqual(float(3.14), 3.14)
 self.assertEqual(float(314), 314.0)
 self.assertEqual(float(" 3.14 "), 3.14)
- self.assertEqual(float(b" 3.14 "), 3.14)
 self.assertRaises(ValueError, float, " 0x3.1 ")
 self.assertRaises(ValueError, float, " -0x3.p-1 ")
 self.assertRaises(ValueError, float, " +0x3.p-1 ")
@@ -43,7 +42,6 @@
 self.assertRaises(ValueError, float, "+.inf")
 self.assertRaises(ValueError, float, ".")
 self.assertRaises(ValueError, float, "-.")
- self.assertRaises(ValueError, float, b"-")
 self.assertRaises(TypeError, float, {})
 self.assertRaisesRegex(TypeError, "not 'dict'", float, {})
 # Lone surrogate
@@ -57,6 +55,42 @@
 float(b'.' + b'1'*1000)
 float('.' + '1'*1000)
 
+ def test_non_numeric_input_types(self):
+ # Test possible non-numeric types for the argument x, including
+ # subclasses of the explicitly documented accepted types.
+ class CustomStr(str): pass
+ class CustomBytes(bytes): pass
+ class CustomByteArray(bytearray): pass
+
+ factories = [
+ bytes,
+ bytearray,
+ lambda b: CustomStr(b.decode()),
+ CustomBytes,
+ CustomByteArray,
+ memoryview,
+ ]
+ try:
+ from array import array
+ except ImportError:
+ pass
+ else:
+ factories.append(lambda b: array('B', b))
+
+ for f in factories:
+ x = f(b" 3.14 ")
+ with self.subTest(type(x)):
+ self.assertEqual(float(x), 3.14)
+ with self.assertRaisesRegex(ValueError, "could not convert"):
+ float(f(b'A' * 0x10))
+
+ def test_float_memoryview(self):
+ self.assertEqual(float(memoryview(b'12.3')[1:4]), 2.3)
+ self.assertEqual(float(memoryview(b'12.3\x00')[1:4]), 2.3)
+ self.assertEqual(float(memoryview(b'12.3 ')[1:4]), 2.3)
+ self.assertEqual(float(memoryview(b'12.3A')[1:4]), 2.3)
+ self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3)
+
 def test_error_message(self):
 testlist = ('\xbd', '123\xbd', ' 123 456 ')
 for s in testlist:
diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
--- a/Lib/test/test_int.py
+++ b/Lib/test/test_int.py
@@ -276,16 +276,40 @@
 class CustomBytes(bytes): pass
 class CustomByteArray(bytearray): pass
 
- values = [b'100',
- bytearray(b'100'),
- CustomStr('100'),
- CustomBytes(b'100'),
- CustomByteArray(b'100')]
+ factories = [
+ bytes,
+ bytearray,
+ lambda b: CustomStr(b.decode()),
+ CustomBytes,
+ CustomByteArray,
+ memoryview,
+ ]
+ try:
+ from array import array
+ except ImportError:
+ pass
+ else:
+ factories.append(lambda b: array('B', b))
 
- for x in values:
- msg = 'x has type %s' % type(x).__name__
- self.assertEqual(int(x), 100, msg=msg)
- self.assertEqual(int(x, 2), 4, msg=msg)
+ for f in factories:
+ x = f(b'100')
+ with self.subTest(type(x)):
+ self.assertEqual(int(x), 100)
+ if isinstance(x, (str, bytes, bytearray)):
+ self.assertEqual(int(x, 2), 4)
+ else:
+ msg = "can't convert non-string"
+ with self.assertRaisesRegex(TypeError, msg):
+ int(x, 2)
+ with self.assertRaisesRegex(ValueError, 'invalid literal'):
+ int(f(b'A' * 0x10))
+
+ def test_int_memoryview(self):
+ self.assertEqual(int(memoryview(b'123')[1:3]), 23)
+ self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23)
+ self.assertEqual(int(memoryview(b'123 ')[1:3]), 23)
+ self.assertEqual(int(memoryview(b'123A')[1:3]), 23)
+ self.assertEqual(int(memoryview(b'1234')[1:3]), 23)
 
 def test_string_float(self):
 self.assertRaises(ValueError, int, '1.2')
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,10 @@
 Core and Builtins
 -----------------
 
+- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()
+ and eval() are passed bytes-like objects. These objects are not
+ necessarily terminated by a null byte, but the functions assumed they were.
+
 - Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that
 was mutated by direct calls of dict methods.
 
diff --git a/Objects/abstract.c b/Objects/abstract.c
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -1309,12 +1309,30 @@
 /* The below check is done in PyLong_FromUnicode(). */
 return PyLong_FromUnicodeObject(o, 10);
 
- if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
+ if (PyBytes_Check(o))
 /* need to do extra error checking that PyLong_FromString()
 * doesn't do. In particular int('9\x005') must raise an
 * exception, not truncate at the null.
 */
- PyObject *result = _PyLong_FromBytes(view.buf, view.len, 10);
+ return _PyLong_FromBytes(PyBytes_AS_STRING(o),
+ PyBytes_GET_SIZE(o), 10);
+
+ if (PyByteArray_Check(o))
+ return _PyLong_FromBytes(PyByteArray_AS_STRING(o),
+ PyByteArray_GET_SIZE(o), 10);
+
+ if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
+ PyObject *result, *bytes;
+
+ /* Copy to NUL-terminated buffer. */
+ bytes = PyBytes_FromStringAndSize((const char *)view.buf, view.len);
+ if (bytes == NULL) {
+ PyBuffer_Release(&view);
+ return NULL;
+ }
+ result = _PyLong_FromBytes(PyBytes_AS_STRING(bytes),
+ PyBytes_GET_SIZE(bytes), 10);
+ Py_DECREF(bytes);
 PyBuffer_Release(&view);
 return result;
 }
diff --git a/Objects/complexobject.c b/Objects/complexobject.c
--- a/Objects/complexobject.c
+++ b/Objects/complexobject.c
@@ -767,7 +767,6 @@
 int got_bracket=0;
 PyObject *s_buffer = NULL;
 Py_ssize_t len;
- Py_buffer view = {NULL, NULL};
 
 if (PyUnicode_Check(v)) {
 s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
@@ -777,10 +776,6 @@
 if (s == NULL)
 goto error;
 }
- else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
- s = (const char *)view.buf;
- len = view.len;
- }
 else {
 PyErr_Format(PyExc_TypeError,
 "complex() argument must be a string or a number, not '%.200s'",
@@ -895,7 +890,6 @@
 if (s-start != len)
 goto parse_error;
 
- PyBuffer_Release(&view);
 Py_XDECREF(s_buffer);
 return complex_subtype_from_doubles(type, x, y);
 
@@ -903,7 +897,6 @@
 PyErr_SetString(PyExc_ValueError,
 "complex() arg is a malformed string");
 error:
- PyBuffer_Release(&view);
 Py_XDECREF(s_buffer);
 return NULL;
 }
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -144,9 +144,24 @@
 return NULL;
 }
 }
+ else if (PyBytes_Check(v)) {
+ s = PyBytes_AS_STRING(v);
+ len = PyBytes_GET_SIZE(v);
+ }
+ else if (PyByteArray_Check(v)) {
+ s = PyByteArray_AS_STRING(v);
+ len = PyByteArray_GET_SIZE(v);
+ }
 else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {
 s = (const char *)view.buf;
 len = view.len;
+ /* Copy to NUL-terminated buffer. */
+ s_buffer = PyBytes_FromStringAndSize(s, len);
+ if (s_buffer == NULL) {
+ PyBuffer_Release(&view);
+ return NULL;
+ }
+ s = PyBytes_AS_STRING(s_buffer);
 }
 else {
 PyErr_Format(PyExc_TypeError,
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -599,20 +599,37 @@
 
 
 static const char *
-source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, Py_buffer *view)
+source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
 {
 const char *str;
 Py_ssize_t size;
+ Py_buffer view;
 
+ *cmd_copy = NULL;
 if (PyUnicode_Check(cmd)) {
 cf->cf_flags |= PyCF_IGNORE_COOKIE;
 str = PyUnicode_AsUTF8AndSize(cmd, &size);
 if (str == NULL)
 return NULL;
 }
- else if (PyObject_GetBuffer(cmd, view, PyBUF_SIMPLE) == 0) {
- str = (const char *)view->buf;
- size = view->len;
+ else if (PyBytes_Check(cmd)) {
+ str = PyBytes_AS_STRING(cmd);
+ size = PyBytes_GET_SIZE(cmd);
+ }
+ else if (PyByteArray_Check(cmd)) {
+ str = PyByteArray_AS_STRING(cmd);
+ size = PyByteArray_GET_SIZE(cmd);
+ }
+ else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
+ /* Copy to NUL-terminated buffer. */
+ *cmd_copy = PyBytes_FromStringAndSize(
+ (const char *)view.buf, view.len);
+ PyBuffer_Release(&view);
+ if (*cmd_copy == NULL) {
+ return NULL;
+ }
+ str = PyBytes_AS_STRING(*cmd_copy);
+ size = PyBytes_GET_SIZE(*cmd_copy);
 }
 else {
 PyErr_Format(PyExc_TypeError,
@@ -624,7 +641,7 @@
 if (strlen(str) != (size_t)size) {
 PyErr_SetString(PyExc_ValueError,
 "source code string cannot contain null bytes");
- PyBuffer_Release(view);
+ Py_CLEAR(*cmd_copy);
 return NULL;
 }
 return str;
@@ -660,7 +677,7 @@
 int dont_inherit, int optimize)
 /*[clinic end generated code: output=31881762c1bb90c4 input=9d53e8cfb3c86414]*/
 {
- Py_buffer view = {NULL, NULL};
+ PyObject *source_copy;
 const char *str;
 int compile_mode = -1;
 int is_ast;
@@ -732,12 +749,12 @@
 goto finally;
 }
 
- str = source_as_string(source, "compile", "string, bytes or AST", &cf, &view);
+ str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
 if (str == NULL)
 goto error;
 
 result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
- PyBuffer_Release(&view);
+ Py_XDECREF(source_copy);
 goto finally;
 
 error:
@@ -812,8 +829,7 @@
 PyObject *locals)
 /*[clinic end generated code: output=7284501fb7b4d666 input=11ee718a8640e527]*/
 {
- PyObject *result, *tmp = NULL;
- Py_buffer view = {NULL, NULL};
+ PyObject *result, *source_copy;
 const char *str;
 PyCompilerFlags cf;
 
@@ -861,7 +877,7 @@
 }
 
 cf.cf_flags = PyCF_SOURCE_IS_UTF8;
- str = source_as_string(source, "eval", "string, bytes or code", &cf, &view);
+ str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
 if (str == NULL)
 return NULL;
 
@@ -870,8 +886,7 @@
 
 (void)PyEval_MergeCompilerFlags(&cf);
 result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
- PyBuffer_Release(&view);
- Py_XDECREF(tmp);
+ Py_XDECREF(source_copy);
 return result;
 }
 
@@ -942,12 +957,13 @@
 v = PyEval_EvalCode(source, globals, locals);
 }
 else {
- Py_buffer view = {NULL, NULL};
+ PyObject *source_copy;
 const char *str;
 PyCompilerFlags cf;
 cf.cf_flags = PyCF_SOURCE_IS_UTF8;
 str = source_as_string(source, "exec",
- "string, bytes or code", &cf, &view);
+ "string, bytes or code", &cf,
+ &source_copy);
 if (str == NULL)
 return NULL;
 if (PyEval_MergeCompilerFlags(&cf))
@@ -955,7 +971,7 @@
 locals, &cf);
 else
 v = PyRun_String(str, Py_file_input, globals, locals);
- PyBuffer_Release(&view);
+ Py_XDECREF(source_copy);
 }
 if (v == NULL)
 return NULL;
-- 
Repository URL: https://hg.python.org/cpython
