[Python-checkins] cpython: streamline normalizer identification a bit

Mon Jan 16 23:23:08 CET 2012

http://hg.python.org/cpython/rev/0677f05d6ece
changeset: 74457:0677f05d6ece
user: Benjamin Peterson <benjamin at python.org>
date: Mon Jan 16 17:22:31 2012 -0500
summary:
 streamline normalizer identification a bit
files:
 Python/ast.c | 80 +++++++++++++++++++++++++++------------
 1 files changed, 55 insertions(+), 25 deletions(-)

diff --git a/Python/ast.c b/Python/ast.c
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -500,6 +500,8 @@
 char *c_encoding; /* source encoding */
 PyArena *c_arena; /* arena for allocating memeory */
 const char *c_filename; /* filename */
+ PyObject *c_normalize; /* Normalization function from unicodedata. */
+ PyObject *c_normalize_args; /* Normalization argument tuple. */
 };
 
 static asdl_seq *seq_for_testlist(struct compiling *, const node *);
@@ -527,36 +529,54 @@
 #define COMP_LISTCOMP 1
 #define COMP_SETCOMP 2
 
+static int
+init_normalization(struct compiling *c)
+{
+ PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
+ if (!m)
+ return 0;
+ c->c_normalize = PyObject_GetAttrString(m, "normalize");
+ Py_DECREF(m);
+ if (!c->c_normalize)
+ return 0;
+ c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None);
+ PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL);
+ if (!c->c_normalize_args) {
+ Py_CLEAR(c->c_normalize);
+ return 0;
+ }
+ return 1;
+}
+
 static identifier
-new_identifier(const char* n, PyArena *arena)
+new_identifier(const char* n, struct compiling *c)
 {
- _Py_IDENTIFIER(normalize);
 PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
 if (!id)
 return NULL;
+ /* PyUnicode_DecodeUTF8 should always return a ready string. */
 assert(PyUnicode_IS_READY(id));
 /* Check whether there are non-ASCII characters in the
 identifier; if so, normalize to NFKC. */
 if (!PyUnicode_IS_ASCII(id)) {
- PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
 PyObject *id2;
- if (!m) {
+ if (!c->c_normalize && !init_normalization(c)) {
 Py_DECREF(id);
 return NULL;
 }
- id2 = _PyObject_CallMethodId(m, &PyId_normalize, "sO", "NFKC", id);
- Py_DECREF(m);
+ PyTuple_SET_ITEM(c->c_normalize_args, 1, id);
+ id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL);
 Py_DECREF(id);
 if (!id2)
 return NULL;
 id = id2;
 }
 PyUnicode_InternInPlace(&id);
- PyArena_AddPyObject(arena, id);
+ PyArena_AddPyObject(c->c_arena, id);
 return id;
 }
 
-#define NEW_IDENTIFIER(n) new_identifier(STR(n), c->c_arena)
+#define NEW_IDENTIFIER(n) new_identifier(STR(n), c)
 
 /* This routine provides an invalid object for the syntax error.
 The outermost routine must unpack this error and create the
@@ -706,13 +726,14 @@
 stmt_ty s;
 node *ch;
 struct compiling c;
+ mod_ty res = NULL;
 
 if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) {
 c.c_encoding = "utf-8";
 if (TYPE(n) == encoding_decl) {
 #if 0
 ast_error(n, "encoding declaration in Unicode string");
- goto error;
+ goto out;
 #endif
 n = CHILD(n, 0);
 }
@@ -725,13 +746,14 @@
 }
 c.c_arena = arena;
 c.c_filename = filename;
+ c.c_normalize = c.c_normalize_args = NULL;
 
 k = 0;
 switch (TYPE(n)) {
 case file_input:
 stmts = asdl_seq_new(num_stmts(n), arena);
 if (!stmts)
- return NULL;
+ goto out;
 for (i = 0; i < NCH(n) - 1; i++) {
 ch = CHILD(n, i);
 if (TYPE(ch) == NEWLINE)
@@ -741,7 +763,7 @@
 if (num == 1) {
 s = ast_for_stmt(&c, ch);
 if (!s)
- goto error;
+ goto out;
 asdl_seq_SET(stmts, k++, s);
 }
 else {
@@ -750,42 +772,44 @@
 for (j = 0; j < num; j++) {
 s = ast_for_stmt(&c, CHILD(ch, j * 2));
 if (!s)
- goto error;
+ goto out;
 asdl_seq_SET(stmts, k++, s);
 }
 }
 }
- return Module(stmts, arena);
+ res = Module(stmts, arena);
+ break;
 case eval_input: {
 expr_ty testlist_ast;
 
 /* XXX Why not comp_for here? */
 testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
 if (!testlist_ast)
- goto error;
- return Expression(testlist_ast, arena);
+ goto out;
+ res = Expression(testlist_ast, arena);
+ break;
 }
 case single_input:
 if (TYPE(CHILD(n, 0)) == NEWLINE) {
 stmts = asdl_seq_new(1, arena);
 if (!stmts)
- goto error;
+ goto out;
 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
 arena));
 if (!asdl_seq_GET(stmts, 0))
- goto error;
- return Interactive(stmts, arena);
+ goto out;
+ res = Interactive(stmts, arena);
 }
 else {
 n = CHILD(n, 0);
 num = num_stmts(n);
 stmts = asdl_seq_new(num, arena);
 if (!stmts)
- goto error;
+ goto out;
 if (num == 1) {
 s = ast_for_stmt(&c, n);
 if (!s)
- goto error;
+ goto out;
 asdl_seq_SET(stmts, 0, s);
 }
 else {
@@ -796,21 +820,27 @@
 break;
 s = ast_for_stmt(&c, CHILD(n, i));
 if (!s)
- goto error;
+ goto out;
 asdl_seq_SET(stmts, i / 2, s);
 }
 }
 
- return Interactive(stmts, arena);
+ res = Interactive(stmts, arena);
+ break;
 }
 default:
 PyErr_Format(PyExc_SystemError,
 "invalid node %d for PyAST_FromNode", TYPE(n));
- goto error;
+ goto out;
 }
- error:
+ out:
+ if (c.c_normalize) {
+ Py_DECREF(c.c_normalize);
+ PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL);
+ Py_DECREF(c.c_normalize_args);
+ }
 ast_error_finish(filename);
- return NULL;
+ return res;
 }
 
 /* Return the AST repr. of the operator represented as syntax (|, ^, etc.)
-- 
Repository URL: http://hg.python.org/cpython
