[Python-checkins] r60092 - in python/trunk: Lib/test/test_builtin.py Lib/test/test_grammar.py Lib/tokenize.py Misc/NEWS Parser/tokenizer.c Python/mystrtoul.c

Sat Jan 19 20:27:06 CET 2008

Author: georg.brandl
Date: Sat Jan 19 20:27:05 2008
New Revision: 60092
Modified:
 python/trunk/Lib/test/test_builtin.py
 python/trunk/Lib/test/test_grammar.py
 python/trunk/Lib/tokenize.py
 python/trunk/Misc/NEWS
 python/trunk/Parser/tokenizer.c
 python/trunk/Python/mystrtoul.c
Log:
Fix #1679: "0x" was taken as a valid integer literal.
Fixes the tokenizer, tokenize.py and int() to reject this.
Patches by Martin Rinehart.
Modified: python/trunk/Lib/test/test_builtin.py
==============================================================================

--- python/trunk/Lib/test/test_builtin.py	(original)
+++ python/trunk/Lib/test/test_builtin.py	Sat Jan 19 20:27:05 2008
@@ -816,6 +816,11 @@
 self.assertEqual(int('0123', 0), 83)
 self.assertEqual(int('0x123', 16), 291)
 
+ # Bug 1679: "0x" is not a valid hex literal
+ self.assertRaises(ValueError, int, "0x", 16)
+ self.assertRaises(ValueError, int, "0x", 0)
+
+
 # SF bug 1334662: int(string, base) wrong answers
 # Various representations of 2**32 evaluated to 0
 # rather than 2**32 in previous versions
Modified: python/trunk/Lib/test/test_grammar.py
==============================================================================
--- python/trunk/Lib/test/test_grammar.py	(original)
+++ python/trunk/Lib/test/test_grammar.py	Sat Jan 19 20:27:05 2008
@@ -30,6 +30,8 @@
 self.assertEquals(0xff, 255)
 self.assertEquals(0377, 255)
 self.assertEquals(2147483647, 017777777777)
+ # "0x" is not a valid literal
+ self.assertRaises(SyntaxError, eval, "0x")
 from sys import maxint
 if maxint == 2147483647:
 self.assertEquals(-2147483647-1, -020000000000)
Modified: python/trunk/Lib/tokenize.py
==============================================================================
--- python/trunk/Lib/tokenize.py	(original)
+++ python/trunk/Lib/tokenize.py	Sat Jan 19 20:27:05 2008
@@ -50,7 +50,7 @@
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
-Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
+Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
 Octnumber = r'0[0-7]*[lL]?'
 Decnumber = r'[1-9]\d*[lL]?'
 Intnumber = group(Hexnumber, Octnumber, Decnumber)
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS	(original)
+++ python/trunk/Misc/NEWS	Sat Jan 19 20:27:05 2008
@@ -12,6 +12,8 @@
 Core and builtins
 -----------------
 
+- Issue #1679: "0x" was taken as a valid integer literal.
+
 - Issue #1865: Bytes as an alias for str and b"" as an alias "" were
 added.
 
Modified: python/trunk/Parser/tokenizer.c
==============================================================================
--- python/trunk/Parser/tokenizer.c	(original)
+++ python/trunk/Parser/tokenizer.c	Sat Jan 19 20:27:05 2008
@@ -1332,7 +1332,14 @@
 				goto imaginary;
 #endif
 			if (c == 'x' || c == 'X') {
+
 				/* Hex */
+				c = tok_nextc(tok);
+				if (!isxdigit(c)) {
+					tok->done = E_TOKEN;
+					tok_backup(tok, c);
+					return ERRORTOKEN;
+				}
 				do {
 					c = tok_nextc(tok);
 				} while (isxdigit(c));
Modified: python/trunk/Python/mystrtoul.c
==============================================================================
--- python/trunk/Python/mystrtoul.c	(original)
+++ python/trunk/Python/mystrtoul.c	Sat Jan 19 20:27:05 2008
@@ -112,27 +112,40 @@
 
 	/* check for leading 0 or 0x for auto-base or base 16 */
 	switch (base) {
-		case 0:		/* look for leading 0, 0x or 0X */
-			if (*str == '0') {
-				++str;
-				if (*str == 'x' || *str == 'X') {
-					++str;
-					base = 16;
+	case 0:		/* look for leading 0, 0x or 0X */
+		if (*str == '0') {
+			++str;
+			if (*str == 'x' || *str == 'X') {
+				/* there must be at least one digit after 0x */
+				if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) {
+					if (ptr)
+						*ptr = str;
+					return 0;
 				}
-				else
-					base = 8;
+				++str;
+				base = 16;
 			}
 			else
-				base = 10;
-			break;
+				base = 8;
+		}
+		else
+			base = 10;
+		break;
 
-		case 16:	/* skip leading 0x or 0X */
-			if (*str == '0') {
+	case 16:	/* skip leading 0x or 0X */
+		if (*str == '0') {
+			++str;
+			if (*str == 'x' || *str == 'X') {
+				/* there must be at least one digit after 0x */
+				if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) {
+					if (ptr)
+						*ptr = str;
+					return 0;
+				}
 				++str;
-				if (*str == 'x' || *str == 'X')
-					++str;
 			}
-			break;
+		}
+		break;
 	}
 
 	/* catch silly bases */
