[Python-checkins] cpython (merge 3.5 -> default): Merge 3.5 (issue #27243)
yury.selivanov
python-checkins at python.org
Thu Jun 9 15:13:31 EDT 2016
https://hg.python.org/cpython/rev/9ff95c30a38e
changeset: 101824:9ff95c30a38e
parent: 101822:2d7dcd8cf928
parent: 101823:93ad47d63b87
user: Yury Selivanov <yury at magic.io>
date: Thu Jun 09 15:13:16 2016 -0400
summary:
Merge 3.5 (issue #27243)
files:
Doc/glossary.rst | 7 +-
Doc/reference/compound_stmts.rst | 2 +-
Doc/reference/datamodel.rst | 48 +++++++++-
Doc/whatsnew/3.5.rst | 13 ++
Include/genobject.h | 3 +
Lib/_collections_abc.py | 4 +-
Lib/asyncio/compat.py | 1 +
Lib/asyncio/streams.py | 6 +
Lib/test/test_coroutines.py | 98 +++++++++++++++----
Lib/test/test_grammar.py | 2 +-
Misc/NEWS | 6 +
Objects/genobject.c | 94 +++++++++++++++++++
Python/ceval.c | 40 +++++++-
13 files changed, 292 insertions(+), 32 deletions(-)
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -76,13 +76,12 @@
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
- Must return an :term:`awaitable` from its :meth:`__aiter__` method,
- which should in turn be resolved in an :term:`asynchronous iterator`
- object. Introduced by :pep:`492`.
+ Must return an :term:`asyncronous iterator` from its
+ :meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
An object that implements :meth:`__aiter__` and :meth:`__anext__`
- methods, that must return :term:`awaitable` objects.
+ methods. ``__anext__`` must return an :term:`awaitable` object.
:keyword:`async for` resolves awaitable returned from asynchronous
iterator's :meth:`__anext__` method until it raises
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -726,7 +726,7 @@
Is semantically equivalent to::
iter = (ITER)
- iter = await type(iter).__aiter__(iter)
+ iter = type(iter).__aiter__(iter)
running = True
while running:
try:
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2360,6 +2360,7 @@
Coroutine objects are automatically closed using the above process when
they are about to be destroyed.
+.. _async-iterators:
Asynchronous Iterators
----------------------
@@ -2372,7 +2373,7 @@
.. method:: object.__aiter__(self)
- Must return an *awaitable* resulting in an *asynchronous iterator* object.
+ Must return an *asynchronous iterator* object.
.. method:: object.__anext__(self)
@@ -2385,7 +2386,7 @@
async def readline(self):
...
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
@@ -2396,6 +2397,49 @@
.. versionadded:: 3.5
+.. note::
+
+ .. versionchanged:: 3.5.2
+ Starting with CPython 3.5.2, ``__aiter__`` can directly return
+ :term:`asynchronous iterators <asynchronous iterator>`. Returning
+ an :term:`awaitable` object will result in a
+ :exc:`PendingDeprecationWarning`.
+
+ The recommended way of writing backwards compatible code in
+ CPython 3.5.x is to continue returning awaitables from
+ ``__aiter__``. If you want to avoid the PendingDeprecationWarning
+ and keep the code backwards compatible, the following decorator
+ can be used::
+
+ import functools
+ import sys
+
+ if sys.version_info < (3, 5, 2):
+ def aiter_compat(func):
+ @functools.wraps(func)
+ async def wrapper(self):
+ return func(self)
+ return wrapper
+ else:
+ def aiter_compat(func):
+ return func
+
+ Example::
+
+ class AsyncIterator:
+
+ @aiter_compat
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ ...
+
+ Starting with CPython 3.6, the :exc:`PendingDeprecationWarning`
+ will be replaced with the :exc:`DeprecationWarning`.
+ In CPython 3.7, returning an awaitable from ``__aiter__`` will
+ result in a :exc:`RuntimeError`.
+
Asynchronous Context Managers
-----------------------------
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -247,6 +247,19 @@
Coroutine functions are intended to be run inside a compatible event loop,
such as the :ref:`asyncio loop <asyncio-event-loop>`.
+
+.. note::
+
+ .. versionchanged:: 3.5.2
+ Starting with CPython 3.5.2, ``__aiter__`` can directly return
+ :term:`asynchronous iterators <asynchronous iterator>`. Returning
+ an :term:`awaitable` object will result in a
+ :exc:`PendingDeprecationWarning`.
+
+ See more details in the :ref:`async-iterators` documentation
+ section.
+
+
.. seealso::
:pep:`492` -- Coroutines with async and await syntax
diff --git a/Include/genobject.h b/Include/genobject.h
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -54,6 +54,9 @@
PyAPI_DATA(PyTypeObject) PyCoro_Type;
PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type;
+PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type;
+PyObject *_PyAIterWrapper_New(PyObject *aiter);
+
#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type)
PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -156,7 +156,7 @@
__slots__ = ()
@abstractmethod
- async def __aiter__(self):
+ def __aiter__(self):
return AsyncIterator()
@classmethod
@@ -176,7 +176,7 @@
"""Return the next item or raise StopAsyncIteration when exhausted."""
raise StopAsyncIteration
- async def __aiter__(self):
+ def __aiter__(self):
return self
@classmethod
diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py
--- a/Lib/asyncio/compat.py
+++ b/Lib/asyncio/compat.py
@@ -4,6 +4,7 @@
PY34 = sys.version_info >= (3, 4)
PY35 = sys.version_info >= (3, 5)
+PY352 = sys.version_info >= (3, 5, 2)
def flatten_list_bytes(list_of_data):
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -689,3 +689,9 @@
if val == b'':
raise StopAsyncIteration
return val
+
+ if compat.PY352:
+ # In Python 3.5.2 and greater, __aiter__ should return
+ # the asynchronous iterator directly.
+ def __aiter__(self):
+ return self
diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py
--- a/Lib/test/test_coroutines.py
+++ b/Lib/test/test_coroutines.py
@@ -1255,8 +1255,9 @@
buffer = []
async def test1():
- async for i1, i2 in AsyncIter():
- buffer.append(i1 + i2)
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i1, i2 in AsyncIter():
+ buffer.append(i1 + i2)
yielded, _ = run_async(test1())
# Make sure that __aiter__ was called only once
@@ -1268,12 +1269,13 @@
buffer = []
async def test2():
nonlocal buffer
- async for i in AsyncIter():
- buffer.append(i[0])
- if i[0] == 20:
- break
- else:
- buffer.append('what?')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AsyncIter():
+ buffer.append(i[0])
+ if i[0] == 20:
+ break
+ else:
+ buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test2())
@@ -1286,12 +1288,13 @@
buffer = []
async def test3():
nonlocal buffer
- async for i in AsyncIter():
- if i[0] > 20:
- continue
- buffer.append(i[0])
- else:
- buffer.append('what?')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AsyncIter():
+ if i[0] > 20:
+ continue
+ buffer.append(i[0])
+ else:
+ buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test3())
@@ -1338,7 +1341,7 @@
def test_for_4(self):
class I:
- async def __aiter__(self):
+ def __aiter__(self):
return self
def __anext__(self):
@@ -1368,8 +1371,9 @@
return 123
async def foo():
- async for i in I():
- print('never going to happen')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in I():
+ print('never going to happen')
with self.assertRaisesRegex(
TypeError,
@@ -1393,7 +1397,7 @@
def __init__(self):
self.i = 0
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
@@ -1417,7 +1421,11 @@
I += 1
I += 1000
- run_async(main())
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ # Test that __aiter__ that returns an asyncronous iterator
+ # directly does not throw any warnings.
+ run_async(main())
self.assertEqual(I, 111011)
self.assertEqual(sys.getrefcount(manager), mrefs_before)
@@ -1472,13 +1480,63 @@
1/0
async def foo():
nonlocal CNT
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AI():
+ CNT += 1
+ CNT += 10
+ with self.assertRaises(ZeroDivisionError):
+ run_async(foo())
+ self.assertEqual(CNT, 0)
+
+ def test_for_8(self):
+ CNT = 0
+ class AI:
+ def __aiter__(self):
+ 1/0
+ async def foo():
+ nonlocal CNT
async for i in AI():
CNT += 1
CNT += 10
with self.assertRaises(ZeroDivisionError):
- run_async(foo())
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ # Test that if __aiter__ raises an exception it propagates
+ # without any kind of warning.
+ run_async(foo())
self.assertEqual(CNT, 0)
+ def test_for_9(self):
+ # Test that PendingDeprecationWarning can safely be converted into
+ # an exception (__aiter__ should not have a chance to raise
+ # a ZeroDivisionError.)
+ class AI:
+ async def __aiter__(self):
+ 1/0
+ async def foo():
+ async for i in AI():
+ pass
+
+ with self.assertRaises(PendingDeprecationWarning):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ run_async(foo())
+
+ def test_for_10(self):
+ # Test that PendingDeprecationWarning can safely be converted into
+ # an exception.
+ class AI:
+ async def __aiter__(self):
+ pass
+ async def foo():
+ async for i in AI():
+ pass
+
+ with self.assertRaises(PendingDeprecationWarning):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ run_async(foo())
+
def test_copy(self):
async def func(): pass
coro = func()
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -1114,7 +1114,7 @@
class Done(Exception): pass
class AIter:
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
raise StopAsyncIteration
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -27,6 +27,12 @@
- Issue #23275: Allow assigning to an empty target list in round brackets:
() = iterable.
+- Issue #27243: Update the __aiter__ protocol: instead of returning
+ an awaitable that resolves to an asynchronous iterator, the asynchronous
+ iterator should be returned directly. Doing the former will trigger a
+ PendingDeprecationWarning.
+
+
Library
-------
diff --git a/Objects/genobject.c b/Objects/genobject.c
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -984,3 +984,97 @@
{
return gen_new_with_qualname(&PyCoro_Type, f, name, qualname);
}
+
+
+/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *aw_aiter;
+} PyAIterWrapper;
+
+
+static PyObject *
+aiter_wrapper_iternext(PyAIterWrapper *aw)
+{
+ PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter);
+ return NULL;
+}
+
+static int
+aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg)
+{
+ Py_VISIT((PyObject *)aw->aw_aiter);
+ return 0;
+}
+
+static void
+aiter_wrapper_dealloc(PyAIterWrapper *aw)
+{
+ _PyObject_GC_UNTRACK((PyObject *)aw);
+ Py_CLEAR(aw->aw_aiter);
+ PyObject_GC_Del(aw);
+}
+
+static PyAsyncMethods aiter_wrapper_as_async = {
+ PyObject_SelfIter, /* am_await */
+ 0, /* am_aiter */
+ 0 /* am_anext */
+};
+
+PyTypeObject _PyAIterWrapper_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ "aiter_wrapper",
+ sizeof(PyAIterWrapper), /* tp_basicsize */
+ 0, /* tp_itemsize */
+ (destructor)aiter_wrapper_dealloc, /* destructor tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ &aiter_wrapper_as_async, /* tp_as_async */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ PyObject_GenericGetAttr, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
+ "A wrapper object for __aiter__ bakwards compatibility.",
+ (traverseproc)aiter_wrapper_traverse, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ PyObject_SelfIter, /* tp_iter */
+ (iternextfunc)aiter_wrapper_iternext, /* tp_iternext */
+ 0, /* tp_methods */
+ 0, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ 0, /* tp_alloc */
+ 0, /* tp_new */
+ PyObject_Del, /* tp_free */
+};
+
+
+PyObject *
+_PyAIterWrapper_New(PyObject *aiter)
+{
+ PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper,
+ &_PyAIterWrapper_Type);
+ if (aw == NULL) {
+ return NULL;
+ }
+ Py_INCREF(aiter);
+ aw->aw_aiter = aiter;
+ _PyObject_GC_TRACK(aw);
+ return (PyObject *)aw;
+}
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1935,8 +1935,9 @@
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
- if (type->tp_as_async != NULL)
+ if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
+ }
if (getter != NULL) {
iter = (*getter)(obj);
@@ -1957,6 +1958,27 @@
goto error;
}
+ if (Py_TYPE(iter)->tp_as_async != NULL &&
+ Py_TYPE(iter)->tp_as_async->am_anext != NULL) {
+
+ /* Starting with CPython 3.5.2 __aiter__ should return
+ asynchronous iterators directly (not awaitables that
+ resolve to asynchronous iterators.)
+
+ Therefore, we check if the object that was returned
+ from __aiter__ has an __anext__ method. If it does,
+ we wrap it in an awaitable that resolves to `iter`.
+
+ See http://bugs.python.org/issue27243 for more
+ details.
+ */
+
+ PyObject *wrapper = _PyAIterWrapper_New(iter);
+ Py_DECREF(iter);
+ SET_TOP(wrapper);
+ DISPATCH();
+ }
+
awaitable = _PyCoro_GetAwaitableIter(iter);
if (awaitable == NULL) {
SET_TOP(NULL);
@@ -1968,9 +1990,23 @@
Py_DECREF(iter);
goto error;
- } else
+ } else {
Py_DECREF(iter);
+ if (PyErr_WarnFormat(
+ PyExc_PendingDeprecationWarning, 1,
+ "'%.100s' implements legacy __aiter__ protocol; "
+ "__aiter__ should return an asynchronous "
+ "iterator, not awaitable",
+ type->tp_name))
+ {
+ /* Warning was converted to an error. */
+ Py_DECREF(awaitable);
+ SET_TOP(NULL);
+ goto error;
+ }
+ }
+
SET_TOP(awaitable);
DISPATCH();
}
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list