Index: Include/pyctype.h =================================================================== --- Include/pyctype.h (revision 82150) +++ Include/pyctype.h (working copy) @@ -11,6 +11,9 @@ extern const unsigned int _Py_ctype_table[256]; +/* Unlike their C counterparts, the following macros are not meant to + * handle an int with any of the values [EOF, 0-UCHAR_MAX]. The argument + * must be a signed/unsigned char. */ #define Py_ISLOWER(c) (_Py_ctype_table[Py_CHARMASK(c)] & PY_CTF_LOWER) #define Py_ISUPPER(c) (_Py_ctype_table[Py_CHARMASK(c)] & PY_CTF_UPPER) #define Py_ISALPHA(c) (_Py_ctype_table[Py_CHARMASK(c)] & PY_CTF_ALPHA) Index: Parser/tokenizer.c =================================================================== --- Parser/tokenizer.c (revision 82150) +++ Parser/tokenizer.c (working copy) @@ -1365,7 +1365,7 @@ goto letter_quote; break; } - while (Py_ISALNUM(c) || c == '_') { + while (c != EOF && (Py_ISALNUM(c) || c == '_')) { c = tok_nextc(tok); } tok_backup(tok, c);