[Python-checkins] bpo-36611: Disable serialno field of debug memory allocators (#12796)

Fri Apr 12 15:54:10 EDT 2019

https://github.com/python/cpython/commit/e8f9acf03484c6c3f163f04a76321419369c28aa
commit: e8f9acf03484c6c3f163f04a76321419369c28aa
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019年04月12日T21:54:06+02:00
summary:
bpo-36611: Disable serialno field of debug memory allocators (#12796)
Omit serialno field from debug hooks on Python memory allocators to
reduce the memory footprint by 5%.
Enable tracemalloc to get the traceback where a memory block has been
allocated when a fatal memory error is logged to decide where to put
a breakpoint.
Compile Python with PYMEM_DEBUG_SERIALNO defined to get back the
field.
files:
A Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst
M Lib/test/test_capi.py
M Objects/obmalloc.c

diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 33c98ac28bc5..31dab6a423e2 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -483,7 +483,7 @@ def test_buffer_overflow(self):
 r" at tail\+1: 0xfd\n"
 r" at tail\+2: 0xfd\n"
 r" .*\n"
- r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
 r" Data at p: cd cd cd .*\n"
 r"\n"
 r"Enable tracemalloc to get the memory block allocation traceback\n"
@@ -499,7 +499,7 @@ def test_api_misuse(self):
 r" 16 bytes originally requested\n"
 r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
 r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
- r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
 r" Data at p: cd cd cd .*\n"
 r"\n"
 r"Enable tracemalloc to get the memory block allocation traceback\n"
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst
new file mode 100644
index 000000000000..f55a9efc5d38
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst	
@@ -0,0 +1,5 @@
+Debug memory allocators: disable serialno field by default from debug hooks on
+Python memory allocators to reduce the memory footprint by 5%. Enable
+:mod:`tracemalloc` to get the traceback where a memory block has been allocated
+when a fatal memory error is logged to decide where to put a breakpoint.
+Compile Python with ``PYMEM_DEBUG_SERIALNO`` defined to get back the field.
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index be43c7a1c2b8..3ee143549d21 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1926,6 +1926,10 @@ _Py_GetAllocatedBlocks(void)
 #define DEADBYTE 0xDD /* dead (newly freed) memory */
 #define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
 
+/* Uncomment this define to add the "serialno" field */
+/* #define PYMEM_DEBUG_SERIALNO */
+
+#ifdef PYMEM_DEBUG_SERIALNO
 static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine. The point is
@@ -1936,9 +1940,16 @@ bumpserialno(void)
 {
 ++serialno;
 }
+#endif
 
 #define SST SIZEOF_SIZE_T
 
+#ifdef PYMEM_DEBUG_SERIALNO
+# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
+#else
+# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
+#endif
+
 /* Read sizeof(size_t) bytes at p as a big-endian size_t. */
 static size_t
 read_size_t(const void *p)
@@ -1967,7 +1978,7 @@ write_size_t(void *p, size_t n)
 }
 }
 
-/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and
+/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and
 fills them with useful stuff, here calling the underlying malloc's result p:
 
 p[0: S]
@@ -1991,6 +2002,9 @@ p[2*S+n+S: 2*S+n+2*S]
 If "bad memory" is detected later, the serial number gives an
 excellent way to set a breakpoint on the next run, to capture the
 instant at which this block was passed out.
+
+If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks
+for 3 * S extra bytes, and omits the last serialno field.
 */
 
 static void *
@@ -2000,21 +2014,24 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
 uint8_t *p; /* base address of malloc'ed pad block */
 uint8_t *data; /* p + 2*SST == pointer to data bytes */
 uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
- size_t total; /* 2 * SST + nbytes + 2 * SST */
+ size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */
 
- if (nbytes > (size_t)PY_SSIZE_T_MAX - 4 * SST) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
 /* integer overflow: can't represent total as a Py_ssize_t */
 return NULL;
 }
- total = nbytes + 4 * SST;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
 
 /* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN]
- * ^--- p ^--- data ^--- tail
+ ^--- p ^--- data ^--- tail
 S: nbytes stored as size_t
 I: API identifier (1 byte)
 F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after)
 C: Clean bytes used later to store actual data
- N: Serial number stored as size_t */
+ N: Serial number stored as size_t
+
+ If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field
+ is omitted. */
 
 if (use_calloc) {
 p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total);
@@ -2027,7 +2044,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
 }
 data = p + 2*SST;
 
+#ifdef PYMEM_DEBUG_SERIALNO
 bumpserialno();
+#endif
 
 /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
 write_size_t(p, nbytes);
@@ -2041,7 +2060,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
 /* at tail, write pad (SST bytes) and serialno (SST bytes) */
 tail = data + nbytes;
 memset(tail, FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
 write_size_t(tail + SST, serialno);
+#endif
 
 return data;
 }
@@ -2081,7 +2102,7 @@ _PyMem_DebugRawFree(void *ctx, void *p)
 
 _PyMem_DebugCheckAddress(api->api_id, p);
 nbytes = read_size_t(q);
- nbytes += 4 * SST;
+ nbytes += PYMEM_DEBUG_EXTRA_BYTES;
 memset(q, DEADBYTE, nbytes);
 api->alloc.free(api->alloc.ctx, q);
 }
@@ -2101,7 +2122,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
 uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
 size_t total; /* 2 * SST + nbytes + 2 * SST */
 size_t original_nbytes;
- size_t block_serialno;
 #define ERASED_SIZE 64
 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */
 
@@ -2110,47 +2130,57 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
 data = (uint8_t *)p;
 head = data - 2*SST;
 original_nbytes = read_size_t(head);
- if (nbytes > (size_t)PY_SSIZE_T_MAX - 4*SST) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
 /* integer overflow: can't represent total as a Py_ssize_t */
 return NULL;
 }
- total = nbytes + 4*SST;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
 
 tail = data + original_nbytes;
- block_serialno = read_size_t(tail + SST);
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t block_serialno = read_size_t(tail + SST);
+#endif
 /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
 ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
 */
 if (original_nbytes <= sizeof(save)) {
 memcpy(save, data, original_nbytes);
- memset(data - 2*SST, DEADBYTE, original_nbytes + 4*SST);
+ memset(data - 2 * SST, DEADBYTE,
+ original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
 }
 else {
 memcpy(save, data, ERASED_SIZE);
- memset(head, DEADBYTE, ERASED_SIZE + 2*SST);
+ memset(head, DEADBYTE, ERASED_SIZE + 2 * SST);
 memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
- memset(tail - ERASED_SIZE, DEADBYTE, ERASED_SIZE + 2*SST);
+ memset(tail - ERASED_SIZE, DEADBYTE,
+ ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
 }
 
 /* Resize and add decorations. */
 r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total);
 if (r == NULL) {
+ /* if realloc() failed: rewrite header and footer which have
+ just been erased */
 nbytes = original_nbytes;
 }
 else {
 head = r;
+#ifdef PYMEM_DEBUG_SERIALNO
 bumpserialno();
 block_serialno = serialno;
+#endif
 }
+ data = head + 2*SST;
 
 write_size_t(head, nbytes);
 head[SST] = (uint8_t)api->api_id;
 memset(head + SST + 1, FORBIDDENBYTE, SST-1);
- data = head + 2*SST;
 
 tail = data + nbytes;
 memset(tail, FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
 write_size_t(tail + SST, block_serialno);
+#endif
 
 /* Restore saved bytes. */
 if (original_nbytes <= sizeof(save)) {
@@ -2170,7 +2200,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
 }
 
 if (nbytes > original_nbytes) {
- /* growing: mark new extra memory clean */
+ /* growing: mark new extra memory clean */
 memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes);
 }
 
@@ -2278,7 +2308,7 @@ _PyObject_DebugDumpAddress(const void *p)
 {
 const uint8_t *q = (const uint8_t *)p;
 const uint8_t *tail;
- size_t nbytes, serial;
+ size_t nbytes;
 int i;
 int ok;
 char id;
@@ -2347,9 +2377,11 @@ _PyObject_DebugDumpAddress(const void *p)
 }
 }
 
- serial = read_size_t(tail + SST);
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t serial = read_size_t(tail + SST);
 fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T
 "u to debug malloc/realloc.\n", serial);
+#endif
 
 if (nbytes > 0) {
 i = 0;
@@ -2575,8 +2607,11 @@ _PyObject_DebugMallocStats(FILE *out)
 quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
 }
 fputc('\n', out);
- if (_PyMem_DebugEnabled())
+#ifdef PYMEM_DEBUG_SERIALNO
+ if (_PyMem_DebugEnabled()) {
 (void)printone(out, "# times object malloc called", serialno);
+ }
+#endif
 (void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
 (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
 (void)printone(out, "# arenas highwater mark", narenas_highwater);
