[Python-checkins] cpython: Issue #14596: The struct.Struct() objects now use more compact implementation.

Fri May 17 09:50:22 CEST 2013

http://hg.python.org/cpython/rev/6707637f68ca
changeset: 83807:6707637f68ca
user: Serhiy Storchaka <storchaka at gmail.com>
date: Fri May 17 10:49:44 2013 +0300
summary:
 Issue #14596: The struct.Struct() objects now use more compact implementation.
files:
 Lib/test/test_struct.py | 9 +-
 Misc/NEWS | 2 +
 Modules/_struct.c | 175 +++++++++++++++------------
 3 files changed, 99 insertions(+), 87 deletions(-)

diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py
--- a/Lib/test/test_struct.py
+++ b/Lib/test/test_struct.py
@@ -8,7 +8,6 @@
 from test import support
 
 ISBIGENDIAN = sys.byteorder == "big"
-IS32BIT = sys.maxsize == 0x7fffffff
 
 integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N'
 byteorders = '', '@', '=', '<', '>', '!'
@@ -538,10 +537,6 @@
 hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2)
 self.assertRaises(struct.error, struct.calcsize, hugecount2)
 
- if IS32BIT:
- def test_crasher(self):
- self.assertRaises(MemoryError, struct.pack, "357913941b", "a")
-
 def test_trailing_counter(self):
 store = array.array('b', b' '*100)
 
@@ -578,7 +573,7 @@
 # The size of 'PyStructObject'
 totalsize = support.calcobjsize('2n3P')
 # The size taken up by the 'formatcode' dynamic array
- totalsize += struct.calcsize('P2n0P') * (number_of_codes + 1)
+ totalsize += struct.calcsize('P3n0P') * (number_of_codes + 1)
 support.check_sizeof(self, struct.Struct(format_str), totalsize)
 
 @support.cpython_only
@@ -589,7 +584,7 @@
 self.check_sizeof('B' * 1234, 1234)
 self.check_sizeof('fd', 2)
 self.check_sizeof('xxxxxxxxxxxxxx', 0)
- self.check_sizeof('100H', 100)
+ self.check_sizeof('100H', 1)
 self.check_sizeof('187s', 1)
 self.check_sizeof('20p', 1)
 self.check_sizeof('0s', 1)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -91,6 +91,8 @@
 Library
 -------
 
+- Issue #14596: The struct.Struct() objects now use more compact implementation.
+
 - Issue #17981: Closed socket on error in SysLogHandler.
 
 - Issue #17964: Fix os.sysconf(): the return type of the C sysconf() function
diff --git a/Modules/_struct.c b/Modules/_struct.c
--- a/Modules/_struct.c
+++ b/Modules/_struct.c
@@ -26,6 +26,7 @@
 const struct _formatdef *fmtdef;
 Py_ssize_t offset;
 Py_ssize_t size;
+ Py_ssize_t repeat;
 } formatcode;
 
 /* Struct object interface */
@@ -1263,7 +1264,7 @@
 const char *s;
 const char *fmt;
 char c;
- Py_ssize_t size, len, num, itemsize;
+ Py_ssize_t size, len, ncodes, num, itemsize;
 
 fmt = PyBytes_AS_STRING(self->s_format);
 
@@ -1272,6 +1273,7 @@
 s = fmt;
 size = 0;
 len = 0;
+ ncodes = 0;
 while ((c = *s++) != '0円') {
 if (Py_ISSPACE(Py_CHARMASK(c)))
 continue;
@@ -1301,9 +1303,9 @@
 
 switch (c) {
 case 's': /* fall through */
- case 'p': len++; break;
+ case 'p': len++; ncodes++; break;
 case 'x': break;
- default: len += num; break;
+ default: len += num; if (num) ncodes++; break;
 }
 
 itemsize = e->size;
@@ -1318,14 +1320,14 @@
 }
 
 /* check for overflow */
- if ((len + 1) > (PY_SSIZE_T_MAX / sizeof(formatcode))) {
+ if ((ncodes + 1) > (PY_SSIZE_T_MAX / sizeof(formatcode))) {
 PyErr_NoMemory();
 return -1;
 }
 
 self->s_size = size;
 self->s_len = len;
- codes = PyMem_MALLOC((len + 1) * sizeof(formatcode));
+ codes = PyMem_MALLOC((ncodes + 1) * sizeof(formatcode));
 if (codes == NULL) {
 PyErr_NoMemory();
 return -1;
@@ -1357,23 +1359,24 @@
 codes->offset = size;
 codes->size = num;
 codes->fmtdef = e;
+ codes->repeat = 1;
 codes++;
 size += num;
 } else if (c == 'x') {
 size += num;
- } else {
- while (--num >= 0) {
- codes->offset = size;
- codes->size = e->size;
- codes->fmtdef = e;
- codes++;
- size += e->size;
- }
+ } else if (num) {
+ codes->offset = size;
+ codes->size = e->size;
+ codes->fmtdef = e;
+ codes->repeat = num;
+ codes++;
+ size += e->size * num;
 }
 }
 codes->fmtdef = NULL;
 codes->offset = size;
 codes->size = 0;
+ codes->repeat = 0;
 
 return 0;
 
@@ -1462,22 +1465,26 @@
 return NULL;
 
 for (code = soself->s_codes; code->fmtdef != NULL; code++) {
- PyObject *v;
 const formatdef *e = code->fmtdef;
 const char *res = startfrom + code->offset;
- if (e->format == 's') {
- v = PyBytes_FromStringAndSize(res, code->size);
- } else if (e->format == 'p') {
- Py_ssize_t n = *(unsigned char*)res;
- if (n >= code->size)
- n = code->size - 1;
- v = PyBytes_FromStringAndSize(res + 1, n);
- } else {
- v = e->unpack(res, e);
+ Py_ssize_t j = code->repeat;
+ while (j--) {
+ PyObject *v;
+ if (e->format == 's') {
+ v = PyBytes_FromStringAndSize(res, code->size);
+ } else if (e->format == 'p') {
+ Py_ssize_t n = *(unsigned char*)res;
+ if (n >= code->size)
+ n = code->size - 1;
+ v = PyBytes_FromStringAndSize(res + 1, n);
+ } else {
+ v = e->unpack(res, e);
+ }
+ if (v == NULL)
+ goto fail;
+ PyTuple_SET_ITEM(result, i++, v);
+ res += code->size;
 }
- if (v == NULL)
- goto fail;
- PyTuple_SET_ITEM(result, i++, v);
 }
 
 return result;
@@ -1716,62 +1723,67 @@
 memset(buf, '0円', soself->s_size);
 i = offset;
 for (code = soself->s_codes; code->fmtdef != NULL; code++) {
- Py_ssize_t n;
- PyObject *v = PyTuple_GET_ITEM(args, i++);
 const formatdef *e = code->fmtdef;
 char *res = buf + code->offset;
- if (e->format == 's') {
- int isstring;
- void *p;
- isstring = PyBytes_Check(v);
- if (!isstring && !PyByteArray_Check(v)) {
- PyErr_SetString(StructError,
- "argument for 's' must be a bytes object");
- return -1;
+ Py_ssize_t j = code->repeat;
+ while (j--) {
+ PyObject *v = PyTuple_GET_ITEM(args, i++);
+ if (e->format == 's') {
+ Py_ssize_t n;
+ int isstring;
+ void *p;
+ isstring = PyBytes_Check(v);
+ if (!isstring && !PyByteArray_Check(v)) {
+ PyErr_SetString(StructError,
+ "argument for 's' must be a bytes object");
+ return -1;
+ }
+ if (isstring) {
+ n = PyBytes_GET_SIZE(v);
+ p = PyBytes_AS_STRING(v);
+ }
+ else {
+ n = PyByteArray_GET_SIZE(v);
+ p = PyByteArray_AS_STRING(v);
+ }
+ if (n > code->size)
+ n = code->size;
+ if (n > 0)
+ memcpy(res, p, n);
+ } else if (e->format == 'p') {
+ Py_ssize_t n;
+ int isstring;
+ void *p;
+ isstring = PyBytes_Check(v);
+ if (!isstring && !PyByteArray_Check(v)) {
+ PyErr_SetString(StructError,
+ "argument for 'p' must be a bytes object");
+ return -1;
+ }
+ if (isstring) {
+ n = PyBytes_GET_SIZE(v);
+ p = PyBytes_AS_STRING(v);
+ }
+ else {
+ n = PyByteArray_GET_SIZE(v);
+ p = PyByteArray_AS_STRING(v);
+ }
+ if (n > (code->size - 1))
+ n = code->size - 1;
+ if (n > 0)
+ memcpy(res + 1, p, n);
+ if (n > 255)
+ n = 255;
+ *res = Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char);
+ } else {
+ if (e->pack(res, v, e) < 0) {
+ if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError))
+ PyErr_SetString(StructError,
+ "long too large to convert to int");
+ return -1;
+ }
 }
- if (isstring) {
- n = PyBytes_GET_SIZE(v);
- p = PyBytes_AS_STRING(v);
- }
- else {
- n = PyByteArray_GET_SIZE(v);
- p = PyByteArray_AS_STRING(v);
- }
- if (n > code->size)
- n = code->size;
- if (n > 0)
- memcpy(res, p, n);
- } else if (e->format == 'p') {
- int isstring;
- void *p;
- isstring = PyBytes_Check(v);
- if (!isstring && !PyByteArray_Check(v)) {
- PyErr_SetString(StructError,
- "argument for 'p' must be a bytes object");
- return -1;
- }
- if (isstring) {
- n = PyBytes_GET_SIZE(v);
- p = PyBytes_AS_STRING(v);
- }
- else {
- n = PyByteArray_GET_SIZE(v);
- p = PyByteArray_AS_STRING(v);
- }
- if (n > (code->size - 1))
- n = code->size - 1;
- if (n > 0)
- memcpy(res + 1, p, n);
- if (n > 255)
- n = 255;
- *res = Py_SAFE_DOWNCAST(n, Py_ssize_t, unsigned char);
- } else {
- if (e->pack(res, v, e) < 0) {
- if (PyLong_Check(v) && PyErr_ExceptionMatches(PyExc_OverflowError))
- PyErr_SetString(StructError,
- "long too large to convert to int");
- return -1;
- }
+ res += code->size;
 }
 }
 
@@ -1907,8 +1919,11 @@
 s_sizeof(PyStructObject *self, void *unused)
 {
 Py_ssize_t size;
+ formatcode *code;
 
- size = sizeof(PyStructObject) + sizeof(formatcode) * (self->s_len + 1);
+ size = sizeof(PyStructObject) + sizeof(formatcode);
+ for (code = self->s_codes; code->fmtdef != NULL; code++)
+ size += sizeof(formatcode);
 return PyLong_FromSsize_t(size);
 }
 
-- 
Repository URL: http://hg.python.org/cpython
