[Python-checkins] [2.7] bpo-31530: Stop crashes when iterating over a file on multiple threads. (#3672)

Fri Nov 10 05:58:59 EST 2017

https://github.com/python/cpython/commit/6401e5671781eb217ee1afb4603cc0d1b0367ae6
commit: 6401e5671781eb217ee1afb4603cc0d1b0367ae6
branch: 2.7
author: Serhiy Storchaka <storchaka at gmail.com>
committer: GitHub <noreply at github.com>
date: 2017年11月10日T12:58:55+02:00
summary:
[2.7] bpo-31530: Stop crashes when iterating over a file on multiple threads. (#3672)
files:
A Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst
M Lib/test/test_file2k.py
M Objects/fileobject.c

diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
index e39ef7042ea..d8966e034e0 100644
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -652,6 +652,38 @@ def io_func():
 self.f.writelines('')
 self._test_close_open_io(io_func)
 
+ def test_iteration_torture(self):
+ # bpo-31530: Crash when concurrently iterate over a file.
+ with open(self.filename, "wb") as fp:
+ for i in xrange(2**20):
+ fp.write(b"0"*50 + b"\n")
+ with open(self.filename, "rb") as f:
+ def iterate():
+ try:
+ for l in f:
+ pass
+ except IOError:
+ pass
+ self._run_workers(iterate, 10)
+
+ def test_iteration_seek(self):
+ # bpo-31530: Crash when concurrently seek and iterate over a file.
+ with open(self.filename, "wb") as fp:
+ for i in xrange(10000):
+ fp.write(b"0"*50 + b"\n")
+ with open(self.filename, "rb") as f:
+ it = iter([1] + [0]*10) # one thread reads, others seek
+ def iterate():
+ try:
+ if next(it):
+ for l in f:
+ pass
+ else:
+ for i in range(100):
+ f.seek(i*100, 0)
+ except IOError:
+ pass
+ self._run_workers(iterate, 10)
 
 @unittest.skipUnless(os.name == 'posix', 'test requires a posix system.')
 class TestFileSignalEINTR(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst
new file mode 100644
index 00000000000..a6cb6c9e9ba
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst	
@@ -0,0 +1,4 @@
+Fixed crashes when iterating over a file on multiple threads.
+seek() and next() methods of file objects now raise an exception during
+concurrent operation on the same file object.
+A lock can be used to prevent the error.
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index 7e07a5376f8..2f63c374d1e 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -430,7 +430,7 @@ close_the_file(PyFileObject *f)
 if (Py_REFCNT(f) > 0) {
 PyErr_SetString(PyExc_IOError,
 "close() called during concurrent "
- "operation on the same file object.");
+ "operation on the same file object");
 } else {
 /* This should not happen unless someone is
 * carelessly playing with the PyFileObject
@@ -438,7 +438,7 @@ close_the_file(PyFileObject *f)
 * pointer. */
 PyErr_SetString(PyExc_SystemError,
 "PyFileObject locking error in "
- "destructor (refcnt <= 0 at close).");
+ "destructor (refcnt <= 0 at close)");
 }
 return NULL;
 }
@@ -762,6 +762,12 @@ file_seek(PyFileObject *f, PyObject *args)
 
 if (f->f_fp == NULL)
 return err_closed();
+ if (f->unlocked_count > 0) {
+ PyErr_SetString(PyExc_IOError,
+ "seek() called during concurrent "
+ "operation on the same file object");
+ return NULL;
+ }
 drop_readahead(f);
 whence = 0;
 if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence))
@@ -2238,6 +2244,7 @@ readahead(PyFileObject *f, Py_ssize_t bufsize)
 {
 Py_ssize_t chunksize;
 
+ assert(f->unlocked_count == 0);
 if (f->f_buf != NULL) {
 if( (f->f_bufend - f->f_bufptr) >= 1)
 return 0;
@@ -2279,6 +2286,12 @@ readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize)
 char *buf;
 Py_ssize_t len;
 
+ if (f->unlocked_count > 0) {
+ PyErr_SetString(PyExc_IOError,
+ "next() called during concurrent "
+ "operation on the same file object");
+ return NULL;
+ }
 if (f->f_buf == NULL)
 if (readahead(f, bufsize) < 0)
 return NULL;
@@ -2692,7 +2705,7 @@ int PyObject_AsFileDescriptor(PyObject *o)
 }
 else {
 PyErr_SetString(PyExc_TypeError,
- "argument must be an int, or have a fileno() method.");
+ "argument must be an int, or have a fileno() method");
 return -1;
 }
 
