[Python-checkins] r65334 - in python/branches/release25-maint: Lib/test/seq_tests.py Lib/test/test_bigmem.py Lib/test/test_strop.py Lib/test/test_support.py Misc/NEWS Modules/gcmodule.c Modules/mmapmodule.c Modules/stropmodule.c Objects/bufferobject.c Objects/longobject.c Objects/stringobject.c Objects/tupleobject.c Objects/unicodeobject.c

Thu Jul 31 19:08:15 CEST 2008

Author: neal.norwitz
Date: Thu Jul 31 19:08:14 2008
New Revision: 65334
Log:
Security patches from Apple: prevent int overflow when allocating memory
Modified:
 python/branches/release25-maint/Lib/test/seq_tests.py
 python/branches/release25-maint/Lib/test/test_bigmem.py
 python/branches/release25-maint/Lib/test/test_strop.py
 python/branches/release25-maint/Lib/test/test_support.py
 python/branches/release25-maint/Misc/NEWS
 python/branches/release25-maint/Modules/gcmodule.c
 python/branches/release25-maint/Modules/mmapmodule.c
 python/branches/release25-maint/Modules/stropmodule.c
 python/branches/release25-maint/Objects/bufferobject.c
 python/branches/release25-maint/Objects/longobject.c
 python/branches/release25-maint/Objects/stringobject.c
 python/branches/release25-maint/Objects/tupleobject.c
 python/branches/release25-maint/Objects/unicodeobject.c
Modified: python/branches/release25-maint/Lib/test/seq_tests.py
==============================================================================

--- python/branches/release25-maint/Lib/test/seq_tests.py	(original)
+++ python/branches/release25-maint/Lib/test/seq_tests.py	Thu Jul 31 19:08:14 2008
@@ -307,11 +307,13 @@
 self.assertEqual(id(s), id(s*1))
 
 def test_bigrepeat(self):
- x = self.type2test([0])
- x *= 2**16
- self.assertRaises(MemoryError, x.__mul__, 2**16)
- if hasattr(x, '__imul__'):
- self.assertRaises(MemoryError, x.__imul__, 2**16)
+ import sys
+ if sys.maxint <= 2147483647:
+ x = self.type2test([0])
+ x *= 2**16
+ self.assertRaises(MemoryError, x.__mul__, 2**16)
+ if hasattr(x, '__imul__'):
+ self.assertRaises(MemoryError, x.__imul__, 2**16)
 
 def test_subscript(self):
 a = self.type2test([10, 11])
Modified: python/branches/release25-maint/Lib/test/test_bigmem.py
==============================================================================
--- python/branches/release25-maint/Lib/test/test_bigmem.py	(original)
+++ python/branches/release25-maint/Lib/test/test_bigmem.py	Thu Jul 31 19:08:14 2008
@@ -1,5 +1,5 @@
 from test import test_support
-from test.test_support import bigmemtest, _1G, _2G
+from test.test_support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
 
 import unittest
 import operator
@@ -54,6 +54,22 @@
 self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
 self.assertEquals(s.strip(), SUBSTR.strip())
 
+ @precisionbigmemtest(size=_2G - 1, memuse=1)
+ def test_center_unicode(self, size):
+ SUBSTR = u' abc def ghi'
+ try:
+ s = SUBSTR.center(size)
+ except OverflowError:
+ pass # acceptable on 32-bit
+ else:
+ self.assertEquals(len(s), size)
+ lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
+ if len(s) % 2:
+ lpadsize += 1
+ self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
+ self.assertEquals(s.strip(), SUBSTR.strip())
+ del s
+
 @bigmemtest(minsize=_2G, memuse=2)
 def test_count(self, size):
 SUBSTR = ' abc def ghi'
@@ -70,10 +86,44 @@
 s = '.' * size
 self.assertEquals(len(s.decode('utf-8')), size)
 
+ def basic_encode_test(self, size, enc, c=u'.', expectedsize=None):
+ if expectedsize is None:
+ expectedsize = size
+
+ s = c * size
+ self.assertEquals(len(s.encode(enc)), expectedsize)
+
 @bigmemtest(minsize=_2G + 2, memuse=3)
 def test_encode(self, size):
- s = u'.' * size
- self.assertEquals(len(s.encode('utf-8')), size)
+ return self.basic_encode_test(size, 'utf-8')
+
+ @precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
+ def test_encode_raw_unicode_escape(self, size):
+ try:
+ return self.basic_encode_test(size, 'raw_unicode_escape')
+ except MemoryError:
+ pass # acceptable on 32-bit
+
+ @precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
+ def test_encode_utf7(self, size):
+ try:
+ return self.basic_encode_test(size, 'utf7')
+ except MemoryError:
+ pass # acceptable on 32-bit
+
+ @precisionbigmemtest(size=_2G-1, memuse=2)
+ def test_decodeascii(self, size):
+ return self.basic_encode_test(size, 'ascii', c='A')
+
+ @precisionbigmemtest(size=_4G / 5, memuse=6+2)
+ def test_unicode_repr_oflw(self, size):
+ try:
+ s = u"\uAAAA"*size
+ r = repr(s)
+ except MemoryError:
+ pass # acceptable on 32-bit
+ else:
+ self.failUnless(s == eval(r))
 
 @bigmemtest(minsize=_2G, memuse=2)
 def test_endswith(self, size):
@@ -459,6 +509,11 @@
 self.assertEquals(s.count('\\'), size)
 self.assertEquals(s.count('0'), size * 2)
 
+ @bigmemtest(minsize=2**32 / 5, memuse=6+2)
+ def test_unicode_repr(self, size):
+ s = u"\uAAAA" * size
+ self.failUnless(len(repr(s)) > size)
+
 # This test is meaningful even with size < 2G, as long as the
 # doubled string is > 2G (but it tests more if both are > 2G :)
 @bigmemtest(minsize=_1G + 2, memuse=3)
@@ -642,6 +697,35 @@
 def test_repeat_large(self, size):
 return self.basic_test_repeat(size)
 
+ @bigmemtest(minsize=_1G - 1, memuse=12)
+ def test_repeat_large_2(self, size):
+ return self.basic_test_repeat(size)
+
+ @precisionbigmemtest(size=_1G - 1, memuse=9)
+ def test_from_2G_generator(self, size):
+ try:
+ t = tuple(xrange(size))
+ except MemoryError:
+ pass # acceptable on 32-bit
+ else:
+ count = 0
+ for item in t:
+ self.assertEquals(item, count)
+ count += 1
+ self.assertEquals(count, size)
+
+ @precisionbigmemtest(size=_1G - 25, memuse=9)
+ def test_from_almost_2G_generator(self, size):
+ try:
+ t = tuple(xrange(size))
+ count = 0
+ for item in t:
+ self.assertEquals(item, count)
+ count += 1
+ self.assertEquals(count, size)
+ except MemoryError:
+ pass # acceptable, expected on 32-bit
+
 # Like test_concat, split in two.
 def basic_test_repr(self, size):
 t = (0,) * size
@@ -957,8 +1041,23 @@
 self.assertEquals(l[:10], [1] * 10)
 self.assertEquals(l[-10:], [5] * 10)
 
+class BufferTest(unittest.TestCase):
+
+ @precisionbigmemtest(size=_1G, memuse=4)
+ def test_repeat(self, size):
+ try:
+ b = buffer("AAAA")*size
+ except MemoryError:
+ pass # acceptable on 32-bit
+ else:
+ count = 0
+ for c in b:
+ self.assertEquals(c, 'A')
+ count += 1
+ self.assertEquals(count, size*4)
+
 def test_main():
- test_support.run_unittest(StrTest, TupleTest, ListTest)
+ test_support.run_unittest(StrTest, TupleTest, ListTest, BufferTest)
 
 if __name__ == '__main__':
 if len(sys.argv) > 1:
Modified: python/branches/release25-maint/Lib/test/test_strop.py
==============================================================================
--- python/branches/release25-maint/Lib/test/test_strop.py	(original)
+++ python/branches/release25-maint/Lib/test/test_strop.py	Thu Jul 31 19:08:14 2008
@@ -115,6 +115,25 @@
 strop.uppercase
 strop.whitespace
 
+ @test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=5)
+ def test_stropjoin_huge_list(self, size):
+ a = "A" * size
+ try:
+ r = strop.join([a, a], a)
+ except OverflowError:
+ pass
+ else:
+ self.assertEquals(len(r), len(a) * 3)
+
+ @test_support.precisionbigmemtest(size=test_support._2G - 1, memuse=1)
+ def test_stropjoin_huge_tup(self, size):
+ a = "A" * size
+ try:
+ r = strop.join((a, a), a)
+ except OverflowError:
+ pass # acceptable on 32-bit
+ else:
+ self.assertEquals(len(r), len(a) * 3)
 
 transtable = '000円001円002円003円004円005円006円007円010円011円012円013円014円015円016円017円020円021円022円023円024円025円026円027円030円031円032円033円034円035円036円037円 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{|}~177円200円201円202円203円204円205円206円207円210円211円212円213円214円215円216円217円220円221円222円223円224円225円226円227円230円231円232円233円234円235円236円237円240円241円242円243円244円245円246円247円250円251円252円253円254円255円256円257円260円261円262円263円264円265円266円267円270円271円272円273円274円275円276円277円300円301円302円303円304円305円306円307円310円311円312円313円314円315円316円317円320円321円322円323円324円325円326円327円330円331円332円333円334円335円336円337円340円341円342円343円344円345円346円347円350円351円352円353円354円355円356円357円360円361円362円363円364円365円366円367円370円371円372円373円374円375円376円377円'
 
Modified: python/branches/release25-maint/Lib/test/test_support.py
==============================================================================
--- python/branches/release25-maint/Lib/test/test_support.py	(original)
+++ python/branches/release25-maint/Lib/test/test_support.py	Thu Jul 31 19:08:14 2008
@@ -33,6 +33,7 @@
 use_resources = None # Flag set to [] by regrtest.py
 max_memuse = 0 # Disable bigmem tests (they will still be run with
 # small sizes, to make sure they work.)
+real_max_memuse = 0
 
 # _original_stdout is meant to hold stdout at the time regrtest began.
 # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
@@ -323,6 +324,7 @@
 _1M = 1024*1024
 _1G = 1024 * _1M
 _2G = 2 * _1G
+_4G = 4 * _1G
 
 # Hack to get at the maximum value an internal index can take.
 class _Dummy:
@@ -333,6 +335,7 @@
 def set_memlimit(limit):
 import re
 global max_memuse
+ global real_max_memuse
 sizes = {
 'k': 1024,
 'm': _1M,
@@ -344,6 +347,7 @@
 if m is None:
 raise ValueError('Invalid memory limit %r' % (limit,))
 memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
+ real_max_memuse = memlimit
 if memlimit > MAX_Py_ssize_t:
 memlimit = MAX_Py_ssize_t
 if memlimit < _2G - 1:
@@ -389,6 +393,27 @@
 return wrapper
 return decorator
 
+def precisionbigmemtest(size, memuse, overhead=5*_1M):
+ def decorator(f):
+ def wrapper(self):
+ if not real_max_memuse:
+ maxsize = 5147
+ else:
+ maxsize = size
+
+ if real_max_memuse and real_max_memuse < maxsize * memuse:
+ if verbose:
+ sys.stderr.write("Skipping %s because of memory "
+ "constraint\n" % (f.__name__,))
+ return
+
+ return f(self, maxsize)
+ wrapper.size = size
+ wrapper.memuse = memuse
+ wrapper.overhead = overhead
+ return wrapper
+ return decorator
+
 def bigaddrspacetest(f):
 """Decorator for tests that fill the address space."""
 def wrapper(self):
Modified: python/branches/release25-maint/Misc/NEWS
==============================================================================
--- python/branches/release25-maint/Misc/NEWS	(original)
+++ python/branches/release25-maint/Misc/NEWS	Thu Jul 31 19:08:14 2008
@@ -12,6 +12,8 @@
 Core and builtins
 -----------------
 
+- Apply security patches from Apple.
+
 - Issue #2620: Overflow checking when allocating or reallocating memory
 was not always being done properly in some python types and extension
 modules. PyMem_MALLOC, PyMem_REALLOC, PyMem_NEW and PyMem_RESIZE have
Modified: python/branches/release25-maint/Modules/gcmodule.c
==============================================================================
--- python/branches/release25-maint/Modules/gcmodule.c	(original)
+++ python/branches/release25-maint/Modules/gcmodule.c	Thu Jul 31 19:08:14 2008
@@ -1318,7 +1318,10 @@
 _PyObject_GC_Malloc(size_t basicsize)
 {
 	PyObject *op;
-	PyGC_Head *g = (PyGC_Head *)PyObject_MALLOC(
+	PyGC_Head *g;
+	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
+		return PyErr_NoMemory();
+	g = (PyGC_Head *)PyObject_MALLOC(
 sizeof(PyGC_Head) + basicsize);
 	if (g == NULL)
 		return PyErr_NoMemory();
@@ -1361,6 +1364,8 @@
 {
 	const size_t basicsize = _PyObject_VAR_SIZE(op->ob_type, nitems);
 	PyGC_Head *g = AS_GC(op);
+	if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
+		return (PyVarObject *)PyErr_NoMemory();
 	g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);
 	if (g == NULL)
 		return (PyVarObject *)PyErr_NoMemory();
Modified: python/branches/release25-maint/Modules/mmapmodule.c
==============================================================================
--- python/branches/release25-maint/Modules/mmapmodule.c	(original)
+++ python/branches/release25-maint/Modules/mmapmodule.c	Thu Jul 31 19:08:14 2008
@@ -223,7 +223,7 @@
 		return(NULL);
 
 	/* silently 'adjust' out-of-range requests */
-	if ((self->pos + num_bytes) > self->size) {
+	if (num_bytes > self->size - self->pos) {
 		num_bytes -= (self->pos+num_bytes) - self->size;
 	}
 	result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
Modified: python/branches/release25-maint/Modules/stropmodule.c
==============================================================================
--- python/branches/release25-maint/Modules/stropmodule.c	(original)
+++ python/branches/release25-maint/Modules/stropmodule.c	Thu Jul 31 19:08:14 2008
@@ -216,6 +216,13 @@
 				return NULL;
 			}
 			slen = PyString_GET_SIZE(item);
+			if (slen > PY_SSIZE_T_MAX - reslen ||
+			 seplen > PY_SSIZE_T_MAX - reslen - seplen) {
+				PyErr_SetString(PyExc_OverflowError,
+						"input too long");
+				Py_DECREF(res);
+				return NULL;
+			}
 			while (reslen + slen + seplen >= sz) {
 				if (_PyString_Resize(&res, sz * 2) < 0)
 					return NULL;
@@ -253,6 +260,14 @@
 			return NULL;
 		}
 		slen = PyString_GET_SIZE(item);
+		if (slen > PY_SSIZE_T_MAX - reslen ||
+		 seplen > PY_SSIZE_T_MAX - reslen - seplen) {
+			PyErr_SetString(PyExc_OverflowError,
+					"input too long");
+			Py_DECREF(res);
+			Py_XDECREF(item);
+			return NULL;
+		}
 		while (reslen + slen + seplen >= sz) {
 			if (_PyString_Resize(&res, sz * 2) < 0) {
 				Py_DECREF(item);
Modified: python/branches/release25-maint/Objects/bufferobject.c
==============================================================================
--- python/branches/release25-maint/Objects/bufferobject.c	(original)
+++ python/branches/release25-maint/Objects/bufferobject.c	Thu Jul 31 19:08:14 2008
@@ -427,6 +427,10 @@
 		count = 0;
 	if (!get_buf(self, &ptr, &size, ANY_BUFFER))
 		return NULL;
+	if (count > PY_SSIZE_T_MAX / size) {
+		PyErr_SetString(PyExc_MemoryError, "result too large");
+		return NULL;
+	}
 	ob = PyString_FromStringAndSize(NULL, size * count);
 	if ( ob == NULL )
 		return NULL;
Modified: python/branches/release25-maint/Objects/longobject.c
==============================================================================
--- python/branches/release25-maint/Objects/longobject.c	(original)
+++ python/branches/release25-maint/Objects/longobject.c	Thu Jul 31 19:08:14 2008
@@ -70,6 +70,8 @@
 		PyErr_NoMemory();
 		return NULL;
 	}
+	/* XXX(nnorwitz): This can overflow --
+	 PyObject_NEW_VAR / _PyObject_VAR_SIZE need to detect overflow */
 	return PyObject_NEW_VAR(PyLongObject, &PyLong_Type, size);
 }
 
Modified: python/branches/release25-maint/Objects/stringobject.c
==============================================================================
--- python/branches/release25-maint/Objects/stringobject.c	(original)
+++ python/branches/release25-maint/Objects/stringobject.c	Thu Jul 31 19:08:14 2008
@@ -75,6 +75,11 @@
 		return (PyObject *)op;
 	}
 
+	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
+		PyErr_SetString(PyExc_OverflowError, "string is too large");
+		return NULL;
+	}
+
 	/* Inline PyObject_NewVar */
 	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
 	if (op == NULL)
@@ -110,7 +115,7 @@
 
 	assert(str != NULL);
 	size = strlen(str);
-	if (size > PY_SSIZE_T_MAX) {
+	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
 		PyErr_SetString(PyExc_OverflowError,
 			"string is too long for a Python string");
 		return NULL;
@@ -971,14 +976,24 @@
 		Py_INCREF(a);
 		return (PyObject *)a;
 	}
+	/* Check that string sizes are not negative, to prevent an
+	 overflow in cases where we are passed incorrectly-created
+	 strings with negative lengths (due to a bug in other code).
+	*/
 	size = a->ob_size + b->ob_size;
-	if (size < 0) {
+	if (a->ob_size < 0 || b->ob_size < 0 ||
+	 a->ob_size > PY_SSIZE_T_MAX - b->ob_size) {
 		PyErr_SetString(PyExc_OverflowError,
 				"strings are too large to concat");
 		return NULL;
 	}
 	 
 	/* Inline PyObject_NewVar */
+	if (size > PY_SSIZE_T_MAX - sizeof(PyStringObject)) {
+		PyErr_SetString(PyExc_OverflowError,
+				"strings are too large to concat");
+		return NULL;
+	}
 	op = (PyStringObject *)PyObject_MALLOC(sizeof(PyStringObject) + size);
 	if (op == NULL)
 		return PyErr_NoMemory();
Modified: python/branches/release25-maint/Objects/tupleobject.c
==============================================================================
--- python/branches/release25-maint/Objects/tupleobject.c	(original)
+++ python/branches/release25-maint/Objects/tupleobject.c	Thu Jul 31 19:08:14 2008
@@ -60,11 +60,12 @@
 		Py_ssize_t nbytes = size * sizeof(PyObject *);
 		/* Check for overflow */
 		if (nbytes / sizeof(PyObject *) != (size_t)size ||
-		 (nbytes += sizeof(PyTupleObject) - sizeof(PyObject *))
-		 <= 0)
+		 (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
 		{
 			return PyErr_NoMemory();
 		}
+		nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);
+
 		op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
 		if (op == NULL)
 			return NULL;
Modified: python/branches/release25-maint/Objects/unicodeobject.c
==============================================================================
--- python/branches/release25-maint/Objects/unicodeobject.c	(original)
+++ python/branches/release25-maint/Objects/unicodeobject.c	Thu Jul 31 19:08:14 2008
@@ -240,6 +240,11 @@
 return unicode_empty;
 }
 
+ /* Ensure we won't overflow the size. */
+ if (length > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) {
+ return (PyUnicodeObject *)PyErr_NoMemory();
+ }
+
 /* Unicode freelist & memory allocation */
 if (unicode_freelist) {
 unicode = unicode_freelist;
@@ -1095,6 +1100,9 @@
 char * out;
 char * start;
 
+ if (cbAllocated / 5 != size)
+ return PyErr_NoMemory();
+
 if (size == 0)
 		return PyString_FromStringAndSize(NULL, 0);
 
@@ -1693,8 +1701,9 @@
 {
 PyObject *v;
 unsigned char *p;
+ Py_ssize_t nsize, bytesize;
 #ifdef Py_UNICODE_WIDE
- int i, pairs;
+ Py_ssize_t i, pairs;
 #else
 const int pairs = 0;
 #endif
@@ -1717,8 +1726,15 @@
 	if (s[i] >= 0x10000)
 	 pairs++;
 #endif
- v = PyString_FromStringAndSize(NULL,
-		 2 * (size + pairs + (byteorder == 0)));
+ /* 2 * (size + pairs + (byteorder == 0)) */
+ if (size > PY_SSIZE_T_MAX ||
+	size > PY_SSIZE_T_MAX - pairs - (byteorder == 0))
+	return PyErr_NoMemory();
+ nsize = (size + pairs + (byteorder == 0));
+ bytesize = nsize * 2;
+ if (bytesize / 2 != nsize)
+	return PyErr_NoMemory();
+ v = PyString_FromStringAndSize(NULL, bytesize);
 if (v == NULL)
 return NULL;
 
@@ -2046,6 +2062,11 @@
 char *p;
 
 static const char *hexdigit = "0123456789abcdef";
+#ifdef Py_UNICODE_WIDE
+ const Py_ssize_t expandsize = 10;
+#else
+ const Py_ssize_t expandsize = 6;
+#endif
 
 /* Initial allocation is based on the longest-possible unichr
 escape.
@@ -2061,13 +2082,12 @@
 escape.
 */
 
+ if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize)
+	return PyErr_NoMemory();
+
 repr = PyString_FromStringAndSize(NULL,
 2
-#ifdef Py_UNICODE_WIDE
- + 10*size
-#else
- + 6*size
-#endif
+ + expandsize*size
 + 1);
 if (repr == NULL)
 return NULL;
@@ -2320,12 +2340,16 @@
 char *q;
 
 static const char *hexdigit = "0123456789abcdef";
-
 #ifdef Py_UNICODE_WIDE
- repr = PyString_FromStringAndSize(NULL, 10 * size);
+ const Py_ssize_t expandsize = 10;
 #else
- repr = PyString_FromStringAndSize(NULL, 6 * size);
+ const Py_ssize_t expandsize = 6;
 #endif
+ 
+ if (size > PY_SSIZE_T_MAX / expandsize)
+	return PyErr_NoMemory();
+ 
+ repr = PyString_FromStringAndSize(NULL, expandsize * size);
 if (repr == NULL)
 return NULL;
 if (size == 0)
@@ -4761,6 +4785,11 @@
 return self;
 }
 
+ if (left > PY_SSIZE_T_MAX - self->length ||
+	right > PY_SSIZE_T_MAX - (left + self->length)) {
+ PyErr_SetString(PyExc_OverflowError, "padded string is too long");
+ return NULL;
+ }
 u = _PyUnicode_New(left + self->length + right);
 if (u) {
 if (left)
