[Python-checkins] cpython (merge 3.3 -> default): Issue #19886: Use better estimated memory requirements for bigmem tests.

Fri Jan 10 12:42:03 CET 2014

http://hg.python.org/cpython/rev/fff1455e1147
changeset: 88389:fff1455e1147
parent: 88386:5a8301002f5a
parent: 88388:2ca26065fb00
user: Serhiy Storchaka <storchaka at gmail.com>
date: Fri Jan 10 13:39:27 2014 +0200
summary:
 Issue #19886: Use better estimated memory requirements for bigmem tests.
Incorrect requirements can cause memory swapping.
files:
 Lib/test/pickletester.py | 12 ++++-----
 Lib/test/test_bz2.py | 2 +-
 Lib/test/test_hashlib.py | 29 ++++++++---------------
 Lib/test/test_marshal.py | 10 ++++----
 Lib/test/test_xml_etree_c.py | 4 +--
 Misc/NEWS | 5 ++++
 6 files changed, 27 insertions(+), 35 deletions(-)

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -21,8 +21,6 @@
 # kind of outer loop.
 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
 
-ascii_char_size = 1
-
 
 # Return True if opcode code appears in the pickle, else False.
 def opcode_in_pickle(code, pickle):
@@ -1643,7 +1641,7 @@
 
 # Binary protocols can serialize longs of up to 2GB-1
 
- @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+ @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
 def test_huge_long_32b(self, size):
 data = 1 << (8 * size)
 try:
@@ -1660,7 +1658,7 @@
 # (older protocols don't have a dedicated opcode for bytes and are
 # too inefficient)
 
- @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+ @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
 def test_huge_bytes_32b(self, size):
 data = b"abcd" * (size // 4)
 try:
@@ -1681,7 +1679,7 @@
 finally:
 data = None
 
- @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
+ @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
 def test_huge_bytes_64b(self, size):
 data = b"acbd" * (size // 4)
 try:
@@ -1711,7 +1709,7 @@
 # All protocols use 1-byte per printable ASCII character; we add another
 # byte because the encoded form has to be copied into the internal buffer.
 
- @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False)
+ @bigmemtest(size=_2G, memuse=8, dry_run=False)
 def test_huge_str_32b(self, size):
 data = "abcd" * (size // 4)
 try:
@@ -1738,7 +1736,7 @@
 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
 # unicode strings however.
 
- @bigmemtest(size=_4G, memuse=2 + ascii_char_size, dry_run=False)
+ @bigmemtest(size=_4G, memuse=8, dry_run=False)
 def test_huge_str_64b(self, size):
 data = "abcd" * (size // 4)
 try:
diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py
--- a/Lib/test/test_bz2.py
+++ b/Lib/test/test_bz2.py
@@ -686,7 +686,7 @@
 self.assertRaises(EOFError, bz2d.decompress, b"anything")
 self.assertRaises(EOFError, bz2d.decompress, b"")
 
- @bigmemtest(size=_4G + 100, memuse=3)
+ @bigmemtest(size=_4G + 100, memuse=3.3)
 def testDecompress4G(self, size):
 # "Test BZ2Decompressor.decompress() with >4GiB input"
 blocksize = 10 * 1024 * 1024
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -256,21 +256,15 @@
 b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
 'd174ab98d277d9f5a5611c2c9f419d9f')
 
- @bigmemtest(size=_4G + 5, memuse=1)
+ @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems')
+ @bigmemtest(size=_4G + 5, memuse=1, dry_run=False)
 def test_case_md5_huge(self, size):
- if size == _4G + 5:
- try:
- self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d')
- except OverflowError:
- pass # 32-bit arch
+ self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d')
 
- @bigmemtest(size=_4G - 1, memuse=1)
+ @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems')
+ @bigmemtest(size=_4G - 1, memuse=1, dry_run=False)
 def test_case_md5_uintmax(self, size):
- if size == _4G - 1:
- try:
- self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
- except OverflowError:
- pass # 32-bit arch
+ self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
 
 # use the three examples from Federal Information Processing Standards
 # Publication 180-1, Secure Hash Standard, 1995 April 17
@@ -379,14 +373,11 @@
 "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+
 "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b")
 
- @bigmemtest(size=_4G + 5, memuse=1)
+ @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems')
+ @bigmemtest(size=_4G + 5, memuse=1, dry_run=False)
 def test_case_sha3_224_huge(self, size):
- if size == _4G + 5:
- try:
- self.check('sha3_224', b'A'*size,
- '58ef60057c9dddb6a87477e9ace5a26f0d9db01881cf9b10a9f8c224')
- except OverflowError:
- pass # 32-bit arch
+ self.check('sha3_224', b'A'*size,
+ '58ef60057c9dddb6a87477e9ace5a26f0d9db01881cf9b10a9f8c224')
 
 def test_gil(self):
 # Check things work fine with an input larger than the size required
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -288,19 +288,19 @@
 def check_unmarshallable(self, data):
 self.assertRaises(ValueError, marshal.dump, data, NullWriter())
 
- @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+ @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
 def test_bytes(self, size):
 self.check_unmarshallable(b'x' * size)
 
- @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+ @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
 def test_str(self, size):
 self.check_unmarshallable('x' * size)
 
- @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False)
+ @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
 def test_tuple(self, size):
 self.check_unmarshallable((None,) * size)
 
- @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size, dry_run=False)
+ @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
 def test_list(self, size):
 self.check_unmarshallable([None] * size)
 
@@ -316,7 +316,7 @@
 def test_frozenset(self, size):
 self.check_unmarshallable(frozenset(range(size)))
 
- @support.bigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+ @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
 def test_bytearray(self, size):
 self.check_unmarshallable(bytearray(size))
 
diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -13,10 +13,8 @@
 
 class MiscTests(unittest.TestCase):
 # Issue #8651.
- @support.bigmemtest(size=support._2G + 100, memuse=1)
+ @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False)
 def test_length_overflow(self, size):
- if size < support._2G + 100:
- self.skipTest("not enough free memory, need at least 2 GB")
 data = b'x' * size
 parser = cET.XMLParser()
 try:
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -58,6 +58,11 @@
 - Issue #18960: IDLE now ignores the source encoding declaration on the second
 line if the first line contains anything except a comment.
 
+Tests
+-----
+
+- Issue #19886: Use better estimated memory requirements for bigmem tests.
+
 Tools/Demos
 -----------
 
-- 
Repository URL: http://hg.python.org/cpython
