changeset: 78208:4cadf91aaddd branch: 2.7 parent: 78187:204be25f24bd user: Antoine Pitrou date: Sat Jul 21 00:52:06 2012 +0200 files: Lib/test/test_codecs.py Misc/NEWS Objects/unicodeobject.c description: Issue #14579: Fix error handling bug in the utf-16 decoder. Patch by Serhiy Storchaka. diff -r 204be25f24bd -r 4cadf91aaddd Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Jul 20 09:48:46 2012 +0100 +++ b/Lib/test/test_codecs.py Sat Jul 21 00:52:06 2012 +0200 @@ -495,7 +495,19 @@ ) def test_errors(self): - self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, "\xff", "strict", True) + tests = [ + (b'\xff', u'\ufffd'), + (b'A\x00Z', u'A\ufffd'), + (b'A\x00B\x00C\x00D\x00Z', u'ABCD\ufffd'), + (b'\x00\xd8', u'\ufffd'), + (b'\x00\xd8A', u'\ufffd'), + (b'\x00\xd8A\x00', u'\ufffdA'), + (b'\x00\xdcA\x00', u'\ufffdA'), + ] + for raw, expected in tests: + self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, + raw, 'strict', True) + self.assertEqual(raw.decode('utf-16le', 'replace'), expected) class UTF16BETest(ReadTest): encoding = "utf-16-be" @@ -516,7 +528,19 @@ ) def test_errors(self): - self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, "\xff", "strict", True) + tests = [ + (b'\xff', u'\ufffd'), + (b'\x00A\xff', u'A\ufffd'), + (b'\x00A\x00B\x00C\x00DZ', u'ABCD\ufffd'), + (b'\xd8\x00', u'\ufffd'), + (b'\xd8\x00\xdc', u'\ufffd'), + (b'\xd8\x00\x00A', u'\ufffdA'), + (b'\xdc\x00\x00A', u'\ufffdA'), + ] + for raw, expected in tests: + self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, + raw, 'strict', True) + self.assertEqual(raw.decode('utf-16be', 'replace'), expected) class UTF8Test(ReadTest): encoding = "utf-8" diff -r 204be25f24bd -r 4cadf91aaddd Misc/NEWS --- a/Misc/NEWS Fri Jul 20 09:48:46 2012 +0100 +++ b/Misc/NEWS Sat Jul 21 00:52:06 2012 +0200 @@ -9,6 +9,9 @@ Core and Builtins ----------------- +- Issue #14579: Fix error handling bug in the utf-16 decoder. Patch by + Serhiy Storchaka. + - Issue #15368: An issue that caused bytecode generation to be non-deterministic when using randomized hashing (-R) has been fixed. diff -r 204be25f24bd -r 4cadf91aaddd Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Jul 20 09:48:46 2012 +0100 +++ b/Objects/unicodeobject.c Sat Jul 21 00:52:06 2012 +0200 @@ -2564,7 +2564,7 @@ } /* UTF-16 code pair: */ - if (q>= e) { + if (e - q < 2) { errmsg = "unexpected end of data"; startinpos = (((const char *)q)-2)-starts; endinpos = ((const char *)e)-starts;