[Python-checkins] bpo-39220: Do not optimise annotation if 'from __future__ import annotations' is used (GH-17866)

Pablo Galindo webhook-mailer at python.org
Wed Mar 18 19:02:14 EDT 2020

• Previous message (by thread): [Python-checkins] bpo-39984: trip_signal() uses PyGILState_GetThisThreadState() (GH-19061)
• Next message (by thread): [Python-checkins] Remove unused variable to fix compiler warning in _threadmodule.c (GH-19064)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

---

https://github.com/python/cpython/commit/d112c600ab3f5e2910345bcc4bfc39439917ed87
commit: d112c600ab3f5e2910345bcc4bfc39439917ed87
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020年03月18日T23:02:09Z
summary:
bpo-39220: Do not optimise annotation if 'from __future__ import annotations' is used (GH-17866)
Do not apply AST-based optimizations if 'from __future__ import annotations' is used in order to
prevent information lost in the final version of the annotations.
files:
A Misc/NEWS.d/next/Core and Builtins/2020-01-06-13-58-37.bpo-39220.KGFovE.rst
M Include/compile.h
M Lib/test/test_future.py
M Python/ast_opt.c
M Python/compile.c
diff --git a/Include/compile.h b/Include/compile.h
index 1cda955c14255..c0c73b29e4d9d 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -83,7 +83,12 @@ PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
 PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
 PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);
 
-PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize);
+typedef struct {
+ int optimize;
+ int ff_features;
+} _PyASTOptimizeState;
+
+PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeState *state);
 
 #ifdef __cplusplus
 }
diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py
index d83c47ef15591..9b88e3f464252 100644
--- a/Lib/test/test_future.py
+++ b/Lib/test/test_future.py
@@ -111,6 +111,10 @@ def f() -> {ann}:
 ...
 def g(arg: {ann}) -> None:
 ...
+ async def f2() -> {ann}:
+ ...
+ async def g2(arg: {ann}) -> None:
+ ...
 var: {ann}
 var2: {ann} = None
 """
@@ -121,9 +125,13 @@ def getActual(self, annotation):
 exec(self.template.format(ann=annotation), {}, scope)
 func_ret_ann = scope['f'].__annotations__['return']
 func_arg_ann = scope['g'].__annotations__['arg']
+ async_func_ret_ann = scope['f2'].__annotations__['return']
+ async_func_arg_ann = scope['g2'].__annotations__['arg']
 var_ann1 = scope['__annotations__']['var']
 var_ann2 = scope['__annotations__']['var2']
 self.assertEqual(func_ret_ann, func_arg_ann)
+ self.assertEqual(func_ret_ann, async_func_ret_ann)
+ self.assertEqual(func_ret_ann, async_func_arg_ann)
 self.assertEqual(func_ret_ann, var_ann1)
 self.assertEqual(func_ret_ann, var_ann2)
 return func_ret_ann
@@ -288,6 +296,7 @@ def test_annotations(self):
 eq('(((a, b)))', '(a, b)')
 eq("(x:=10)")
 eq("f'{(x:=10):=10}'")
+ eq("1 + 2 + 3")
 
 def test_fstring_debug_annotations(self):
 # f-strings with '=' don't round trip very well, so set the expected
@@ -299,6 +308,5 @@ def test_fstring_debug_annotations(self):
 self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'")
 self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'")
 
-
 if __name__ == "__main__":
 unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-13-58-37.bpo-39220.KGFovE.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-13-58-37.bpo-39220.KGFovE.rst
new file mode 100644
index 0000000000000..617d35e1a1ebb
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-13-58-37.bpo-39220.KGFovE.rst	
@@ -0,0 +1,2 @@
+Do not optimize annotations if 'from __future__ import annotations' is used.
+Patch by Pablo Galindo.
diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index 7a2b6e6aab11a..1393c3473bf54 100644
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -35,7 +35,7 @@ unary_not(PyObject *v)
 }
 
 static int
-fold_unaryop(expr_ty node, PyArena *arena, int optimize)
+fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
 {
 expr_ty arg = node->v.UnaryOp.operand;
 
@@ -212,7 +212,7 @@ safe_mod(PyObject *v, PyObject *w)
 }
 
 static int
-fold_binop(expr_ty node, PyArena *arena, int optimize)
+fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
 {
 expr_ty lhs, rhs;
 lhs = node->v.BinOp.left;
@@ -294,7 +294,7 @@ make_const_tuple(asdl_seq *elts)
 }
 
 static int
-fold_tuple(expr_ty node, PyArena *arena, int optimize)
+fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
 {
 PyObject *newval;
 
@@ -306,7 +306,7 @@ fold_tuple(expr_ty node, PyArena *arena, int optimize)
 }
 
 static int
-fold_subscr(expr_ty node, PyArena *arena, int optimize)
+fold_subscr(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
 {
 PyObject *newval;
 expr_ty arg, idx;
@@ -331,7 +331,7 @@ fold_subscr(expr_ty node, PyArena *arena, int optimize)
 in "for" loop and comprehensions.
 */
 static int
-fold_iter(expr_ty arg, PyArena *arena, int optimize)
+fold_iter(expr_ty arg, PyArena *arena, _PyASTOptimizeState *state)
 {
 PyObject *newval;
 if (arg->kind == List_kind) {
@@ -364,7 +364,7 @@ fold_iter(expr_ty arg, PyArena *arena, int optimize)
 }
 
 static int
-fold_compare(expr_ty node, PyArena *arena, int optimize)
+fold_compare(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
 {
 asdl_int_seq *ops;
 asdl_seq *args;
@@ -378,28 +378,28 @@ fold_compare(expr_ty node, PyArena *arena, int optimize)
 i = asdl_seq_LEN(ops) - 1;
 int op = asdl_seq_GET(ops, i);
 if (op == In || op == NotIn) {
- if (!fold_iter((expr_ty)asdl_seq_GET(args, i), arena, optimize)) {
+ if (!fold_iter((expr_ty)asdl_seq_GET(args, i), arena, state)) {
 return 0;
 }
 }
 return 1;
 }
 
-static int astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_arg(arg_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_);
-static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_);
+static int astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_arguments(arguments_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_comprehension(comprehension_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_keyword(keyword_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
+static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
 #define CALL(FUNC, TYPE, ARG) \
- if (!FUNC((ARG), ctx_, optimize_)) \
+ if (!FUNC((ARG), ctx_, state)) \
 return 0;
 
 #define CALL_OPT(FUNC, TYPE, ARG) \
- if ((ARG) != NULL && !FUNC((ARG), ctx_, optimize_)) \
+ if ((ARG) != NULL && !FUNC((ARG), ctx_, state)) \
 return 0;
 
 #define CALL_SEQ(FUNC, TYPE, ARG) { \
@@ -407,7 +407,7 @@ static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int opti
 asdl_seq *seq = (ARG); /* avoid variable capture */ \
 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
 TYPE elt = (TYPE)asdl_seq_GET(seq, i); \
- if (elt != NULL && !FUNC(elt, ctx_, optimize_)) \
+ if (elt != NULL && !FUNC(elt, ctx_, state)) \
 return 0; \
 } \
 }
@@ -417,13 +417,13 @@ static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int opti
 asdl_int_seq *seq = (ARG); /* avoid variable capture */ \
 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
 TYPE elt = (TYPE)asdl_seq_GET(seq, i); \
- if (!FUNC(elt, ctx_, optimize_)) \
+ if (!FUNC(elt, ctx_, state)) \
 return 0; \
 } \
 }
 
 static int
-astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
+astfold_body(asdl_seq *stmts, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 int docstring = _PyAST_GetDocString(stmts) != NULL;
 CALL_SEQ(astfold_stmt, stmt_ty, stmts);
@@ -445,7 +445,7 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_)
+astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 switch (node_->kind) {
 case Module_kind:
@@ -464,7 +464,7 @@ astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_)
+astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 switch (node_->kind) {
 case BoolOp_kind:
@@ -563,7 +563,7 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_)
 break;
 case Name_kind:
 if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
- return make_const(node_, PyBool_FromLong(!optimize_), ctx_);
+ return make_const(node_, PyBool_FromLong(!state->optimize), ctx_);
 }
 break;
 default:
@@ -573,14 +573,14 @@ astfold_expr(expr_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_keyword(keyword_ty node_, PyArena *ctx_, int optimize_)
+astfold_keyword(keyword_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 CALL(astfold_expr, expr_ty, node_->value);
 return 1;
 }
 
 static int
-astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_)
+astfold_comprehension(comprehension_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 CALL(astfold_expr, expr_ty, node_->target);
 CALL(astfold_expr, expr_ty, node_->iter);
@@ -591,7 +591,7 @@ astfold_comprehension(comprehension_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_)
+astfold_arguments(arguments_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 CALL_SEQ(astfold_arg, arg_ty, node_->posonlyargs);
 CALL_SEQ(astfold_arg, arg_ty, node_->args);
@@ -604,27 +604,33 @@ astfold_arguments(arguments_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_arg(arg_ty node_, PyArena *ctx_, int optimize_)
+astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
- CALL_OPT(astfold_expr, expr_ty, node_->annotation);
+ if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
+ CALL_OPT(astfold_expr, expr_ty, node_->annotation);
+ }
 return 1;
 }
 
 static int
-astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
+astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 switch (node_->kind) {
 case FunctionDef_kind:
 CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args);
 CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body);
 CALL_SEQ(astfold_expr, expr_ty, node_->v.FunctionDef.decorator_list);
- CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns);
+ if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
+ CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns);
+ }
 break;
 case AsyncFunctionDef_kind:
 CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args);
 CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body);
 CALL_SEQ(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.decorator_list);
- CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns);
+ if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
+ CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns);
+ }
 break;
 case ClassDef_kind:
 CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.bases);
@@ -648,7 +654,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
 break;
 case AnnAssign_kind:
 CALL(astfold_expr, expr_ty, node_->v.AnnAssign.target);
- CALL(astfold_expr, expr_ty, node_->v.AnnAssign.annotation);
+ if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) {
+ CALL(astfold_expr, expr_ty, node_->v.AnnAssign.annotation);
+ }
 CALL_OPT(astfold_expr, expr_ty, node_->v.AnnAssign.value);
 break;
 case For_kind:
@@ -707,7 +715,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_)
+astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 switch (node_->kind) {
 case ExceptHandler_kind:
@@ -721,7 +729,7 @@ astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int optimize_)
 }
 
 static int
-astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_)
+astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
 {
 CALL(astfold_expr, expr_ty, node_->context_expr);
 CALL_OPT(astfold_expr, expr_ty, node_->optional_vars);
@@ -734,9 +742,9 @@ astfold_withitem(withitem_ty node_, PyArena *ctx_, int optimize_)
 #undef CALL_INT_SEQ
 
 int
-_PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize)
+_PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state)
 {
- int ret = astfold_mod(mod, arena, optimize);
+ int ret = astfold_mod(mod, arena, state);
 assert(ret || PyErr_Occurred());
 return ret;
 }
diff --git a/Python/compile.c b/Python/compile.c
index 486b7bbc017e2..a8ab873f319f9 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -354,7 +354,11 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
 c.c_nestlevel = 0;
 c.c_do_not_emit_bytecode = 0;
 
- if (!_PyAST_Optimize(mod, arena, c.c_optimize)) {
+ _PyASTOptimizeState state;
+ state.optimize = c.c_optimize;
+ state.ff_features = merged;
+
+ if (!_PyAST_Optimize(mod, arena, &state)) {
 goto finally;
 }
 

---

• Previous message (by thread): [Python-checkins] bpo-39984: trip_signal() uses PyGILState_GetThisThreadState() (GH-19061)
• Next message (by thread): [Python-checkins] Remove unused variable to fix compiler warning in _threadmodule.c (GH-19064)
• Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

---

More information about the Python-checkins mailing list