homepage

This error occurred on "x86 Ubuntu 3.x" buildbot.
This is the 1st test on this run.
http://www.python.org/dev/buildbot/all/builders/x86%20Ubuntu%203.x/builds/1699/steps/test/logs/stdio
./python -Wd -E -bb ./Lib/test/regrtest.py -uall -rwW -l 
== CPython 3.2a1+ (py3k:83951, Aug 11 2010, 15:26:40) [GCC 4.2.4 (Ubuntu 4.2.4-1ubuntu4)]
== Linux-2.6.31.5-linode21-i686-with-debian-lenny-sid little-endian
== /srv/buildbot/buildarea/3.x.bolen-ubuntu/build/build/test_python_8258
Using random seed 426296
[ 1/346] test_multiprocessing
Process Process-22:
Traceback (most recent call last):
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/process.py", line 233, in _bootstrap
 self.run()
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/process.py", line 88, in run
 self._target(*self._args, **self._kwargs)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/test/test_multiprocessing.py", line 1253, in _putter
 manager.connect()
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/managers.py", line 478, in connect
 dispatch(conn, None, 'dummy')
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/managers.py", line 79, in dispatch
 kind, result = c.recv()
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/connection.py", line 408, in recv
 s = self._conn.recv_bytes()
EOFError
test test_multiprocessing failed -- Traceback (most recent call last):
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 486, in set_data
 with _closing(_io.FileIO(path, 'wb')) as file:
IOError: [Errno 2] No such file or directory: '/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/xmlrpc/__pycache__/__init__.cpython-32.pyc'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/test/test_multiprocessing.py", line 1266, in test_rapid_restart
 queue = manager.get_queue()
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/managers.py", line 644, in temp
 token, exp = self._create(typeid, *args, **kwds)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/managers.py", line 542, in _create
 conn = self._Client(self._address, authkey=self._authkey)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/multiprocessing/connection.py", line 427, in XmlClient
 import xmlrpc.client as xmlrpclib
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 450, in load_module
 return self._load_module(fullname)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 155, in decorated
 return fxn(self, module, *args, **kwargs)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 344, in _load_module
 code_object = self.get_code(name)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 437, in get_code
 self.set_data(bytecode_path, data)
 File "/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/importlib/_bootstrap.py", line 498, in set_data
 _os.mkdir(directory)
OSError: [Errno 17] File exists: '/srv/buildbot/buildarea/3.x.bolen-ubuntu/build/Lib/xmlrpc/__pycache__'
Re-running test test_multiprocessing in verbose mode
...
----------------------------------------------------------------------
Ran 127 tests in 34.091s
OK (skipped=9)

I succeeded to reproduce one time on Debian Lenny (64 bits).
~ $ make pycremoval
find . -name '*.py[co]' -exec rm -f {} ';'
find . -name '__pycache__' -exec rmdir {} '+'
~ $ ./python -Wd -E -bb ./Lib/test/regrtest.py -uall -rwW -l test_multiprocessing
Using random seed 8192992
[1/1] test_multiprocessing
Process Process-22:
Traceback (most recent call last):
 File "./Lib/multiprocessing/process.py", line 233, in _bootstrap
 self.run()
 File "./Lib/multiprocessing/process.py", line 88, in run
 self._target(*self._args, **self._kwargs)
 File "./Lib/test/test_multiprocessing.py", line 1253, in _putter
 manager.connect()
 File "./Lib/multiprocessing/managers.py", line 478, in connect
 dispatch(conn, None, 'dummy')
 File "./Lib/multiprocessing/managers.py", line 79, in dispatch
 kind, result = c.recv()
 File "./Lib/multiprocessing/connection.py", line 408, in recv
 s = self._conn.recv_bytes()
EOFError
test test_multiprocessing failed -- Traceback (most recent call last):
 File "./Lib/importlib/_bootstrap.py", line 486, in set_data
 with _closing(_io.FileIO(path, 'wb')) as file:
IOError: [Errno 2] No such file or directory: './Lib/email/__pycache__/__init__.cpython-32.pyc'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
 File "./Lib/test/test_multiprocessing.py", line 1266, in test_rapid_restart
 queue = manager.get_queue()
 File "./Lib/multiprocessing/managers.py", line 644, in temp
 token, exp = self._create(typeid, *args, **kwds)
 File "./Lib/multiprocessing/managers.py", line 542, in _create
 conn = self._Client(self._address, authkey=self._authkey)
 File "./Lib/multiprocessing/connection.py", line 427, in XmlClient
 import xmlrpc.client as xmlrpclib
 File "./Lib/xmlrpc/client.py", line 137, in <module>
 import http.client
 File "./Lib/http/client.py", line 69, in <module>
 import email.parser
 File "./Lib/importlib/_bootstrap.py", line 450, in load_module
 return self._load_module(fullname)
 File "./Lib/importlib/_bootstrap.py", line 155, in decorated
 return fxn(self, module, *args, **kwargs)
 File "./Lib/importlib/_bootstrap.py", line 344, in _load_module
 code_object = self.get_code(name)
 File "./Lib/importlib/_bootstrap.py", line 437, in get_code
 self.set_data(bytecode_path, data)
 File "./Lib/importlib/_bootstrap.py", line 498, in set_data
 _os.mkdir(directory)
OSError: [Errno 17] File exists: './Lib/email/__pycache__'
...
But next attempts were successful.

I think what I need to do is greatly simplify the directory creation code in set_data and make it much more robust against potential race conditions against other Python processes. I think as long as I just stop trying to make a directory when it is found to exist this should be okay. Might also want to file lock the bytecode file as import.c does through open_exclusive.

Fixed in r84271. I will open another issue for the locking stuff.

I was just playing around with the changes you made and it seems you've overlooked a small error/typo around line 502:
502 : 	 	 	n elif errno == errno.EACCES: 
quick fix:
502 : 	 	 	n elif exc.errno == errno.EACCES: 
Hope this helps.

Fixed in r84304. Thanks for catching that, Łukasz.

This one looks very similar (r84315 on i386 Ubuntu 3.x buildbot):
./python -Wd -E -bb ./Lib/test/regrtest.py -uall -rwW -l 
== CPython 3.2a1+ (py3k:84315, Aug 25 2010, 17:07:54) [GCC 4.6.0 20100814 (experimental) [trunk revision 163239]]
== Linux-2.6.32-24-server-i686-with-debian-squeeze-sid little-endian
== /scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/build/test_python_17577
Using random seed 4479416
[ 1/346] test_thread
[ 2/346] test_hash
[ 3/346] test_zipfile64
test_zipfile64 skipped -- test requires loads of disk-space bytes and a long time to run
[ 4/346] test_dbm_gnu
test_dbm_gnu skipped -- No module named _gdbm
[ 5/346] test_funcattrs
[ 6/346] test_http_cookies
[ 7/346] test_multiprocessing
Process Process-24:
Traceback (most recent call last):
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/process.py", line 233, in _bootstrap
 self.run()
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/process.py", line 88, in run
 self._target(*self._args, **self._kwargs)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/test/test_multiprocessing.py", line 1260, in _putter
 manager.connect()
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/managers.py", line 477, in connect
 conn = Client(self._address, authkey=self._authkey)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/connection.py", line 427, in XmlClient
 import xmlrpc.client as xmlrpclib
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 450, in load_module
 return self._load_module(fullname)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 155, in decorated
 return fxn(self, module, *args, **kwargs)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 344, in _load_module
 code_object = self.get_code(name)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 437, in get_code
 self.set_data(bytecode_path, data)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 495, in set_data
 _os.mkdir(parent)
OSError: [Errno 17] File exists: '/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/xmlrpc/__pycache__'
test test_multiprocessing failed -- Traceback (most recent call last):
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/test/test_multiprocessing.py", line 1273, in test_rapid_restart
 queue = manager.get_queue()
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/managers.py", line 644, in temp
 token, exp = self._create(typeid, *args, **kwds)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/managers.py", line 542, in _create
 conn = self._Client(self._address, authkey=self._authkey)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/multiprocessing/connection.py", line 427, in XmlClient
 import xmlrpc.client as xmlrpclib
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 450, in load_module
 return self._load_module(fullname)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 155, in decorated
 return fxn(self, module, *args, **kwargs)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 344, in _load_module
 code_object = self.get_code(name)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 437, in get_code
 self.set_data(bytecode_path, data)
 File "/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/importlib/_bootstrap.py", line 495, in set_data
 _os.mkdir(parent)
OSError: [Errno 17] File exists: '/scratch/pybot-buildarea/3.x.klose-ubuntu-i386/build/Lib/xmlrpc/__pycache__'
http://www.python.org/dev/buildbot/builders/i386 Ubuntu 3.x/builds/2005

I Had the same issue while using the lib. I had to replace every: IOError with (IOError, OSError) to catch both and just then compare exc.errno.
Also in your r84271 commit, on line 507 you did:
507 : 	with _io.FileIO(path, 'wb') as file: 
shouldn't it be:
507 : 	with _closing(_io.FileIO(path, 'wb')) as file: 
as in previous commit? In case of an error, you're not closing the file properly. Or am I reading that wrong?
Cheers.

Łukasz,
_closing is not necessary on FileIO instances. The class already declares an __exit__ method which takes care of closing file.
>>> import io
>>> f = io.FileIO('/tmp/test_closing', 'wb')
>>> f.closed
False
>>> f.__exit__()
>>> f.closed
True
Since both IOError and OSError are direct subclasses of EnvironmentError, we can use this in the except clause.

Patch.

r84329 has the fix. I decided to catch IOError and OSError separately rather than EnvironmentError as a blanket 'except' clause; explicit is better than implicit and all.