[Python-checkins] cpython (2.7): Issue #22410: Module level functions in the re module now cache compiled

Fri Oct 31 00:03:58 CET 2014

https://hg.python.org/cpython/rev/6d2788f9b20a
changeset: 93291:6d2788f9b20a
branch: 2.7
parent: 93280:48825a84f87e
user: Serhiy Storchaka <storchaka at gmail.com>
date: Fri Oct 31 00:53:19 2014 +0200
summary:
 Issue #22410: Module level functions in the re module now cache compiled
locale-dependent regular expressions taking into account the locale.
files:
 Lib/re.py | 16 ++++++++++---
 Lib/test/test_re.py | 37 +++++++++++++++++++++++++++++++++
 Misc/NEWS | 3 ++
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/Lib/re.py b/Lib/re.py
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -104,6 +104,7 @@
 import sys
 import sre_compile
 import sre_parse
+import _locale
 
 # public symbols
 __all__ = [ "match", "search", "sub", "subn", "split", "findall",
@@ -229,9 +230,12 @@
 bypass_cache = flags & DEBUG
 if not bypass_cache:
 cachekey = (type(key[0]),) + key
- p = _cache.get(cachekey)
- if p is not None:
- return p
+ try:
+ p, loc = _cache[cachekey]
+ if loc is None or loc == _locale.setlocale(_locale.LC_CTYPE):
+ return p
+ except KeyError:
+ pass
 if isinstance(pattern, _pattern_type):
 if flags:
 raise ValueError('Cannot process flags argument with a compiled pattern')
@@ -245,7 +249,11 @@
 if not bypass_cache:
 if len(_cache) >= _MAXCACHE:
 _cache.clear()
- _cache[cachekey] = p
+ if p.flags & LOCALE:
+ loc = _locale.setlocale(_locale.LC_CTYPE)
+ else:
+ loc = None
+ _cache[cachekey] = p, loc
 return p
 
 def _compile_repl(*key):
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -1,6 +1,7 @@
 from test.test_support import verbose, run_unittest, import_module
 from test.test_support import precisionbigmemtest, _2G, cpython_only
 from test.test_support import captured_stdout, have_unicode, requires_unicode, u
+import locale
 import re
 from re import Scanner
 import sre_constants
@@ -975,6 +976,42 @@
 self.assertEqual(re.match("(foo)", "foo").group(1L), "foo")
 self.assertRaises(IndexError, re.match("", "").group, sys.maxint + 1)
 
+ def test_locale_caching(self):
+ # Issue #22410
+ oldlocale = locale.setlocale(locale.LC_CTYPE)
+ self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale)
+ for loc in 'en_US.iso88591', 'en_US.utf8':
+ try:
+ locale.setlocale(locale.LC_CTYPE, loc)
+ except locale.Error:
+ # Unsupported locale on this system
+ self.skipTest('test needs %s locale' % loc)
+
+ re.purge()
+ self.check_en_US_iso88591()
+ self.check_en_US_utf8()
+ re.purge()
+ self.check_en_US_utf8()
+ self.check_en_US_iso88591()
+
+ def check_en_US_iso88591(self):
+ locale.setlocale(locale.LC_CTYPE, 'en_US.iso88591')
+ self.assertTrue(re.match(b'\xc5\xe5', b'\xc5\xe5', re.L|re.I))
+ self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))
+ self.assertTrue(re.match(b'\xe5', b'\xc5', re.L|re.I))
+ self.assertTrue(re.match(b'(?Li)\xc5\xe5', b'\xc5\xe5'))
+ self.assertTrue(re.match(b'(?Li)\xc5', b'\xe5'))
+ self.assertTrue(re.match(b'(?Li)\xe5', b'\xc5'))
+
+ def check_en_US_utf8(self):
+ locale.setlocale(locale.LC_CTYPE, 'en_US.utf8')
+ self.assertTrue(re.match(b'\xc5\xe5', b'\xc5\xe5', re.L|re.I))
+ self.assertIsNone(re.match(b'\xc5', b'\xe5', re.L|re.I))
+ self.assertIsNone(re.match(b'\xe5', b'\xc5', re.L|re.I))
+ self.assertTrue(re.match(b'(?Li)\xc5\xe5', b'\xc5\xe5'))
+ self.assertIsNone(re.match(b'(?Li)\xc5', b'\xe5'))
+ self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5'))
+
 
 def run_re_tests():
 from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -37,6 +37,9 @@
 Library
 -------
 
+- Issue #22410: Module level functions in the re module now cache compiled
+ locale-dependent regular expressions taking into account the locale.
+
 - Issue #8876: distutils now falls back to copying files when hard linking
 doesn't work. This allows use with special filesystems such as VirtualBox
 shared folders.
-- 
Repository URL: https://hg.python.org/cpython
