[Python-checkins] cpython: Close #19746: expose unittest discovery errors on TestLoader.errors

Mon Oct 20 02:24:33 CEST 2014

https://hg.python.org/cpython/rev/e906e23931fa
changeset: 93143:e906e23931fa
user: Robert Collins <rbtcollins at hp.com>
date: Mon Oct 20 13:24:05 2014 +1300
summary:
 Close #19746: expose unittest discovery errors on TestLoader.errors
This makes it possible to examine the errors from unittest discovery
without executing the test suite - important when the test suite may
be very large, or when enumerating the test ids from a test suite.
files:
 Doc/library/unittest.rst | 14 +++++++
 Lib/unittest/loader.py | 33 ++++++++++++----
 Lib/unittest/test/test_discovery.py | 14 +++++++
 Lib/unittest/test/test_loader.py | 14 +++++++
 Misc/NEWS | 4 ++
 5 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -1552,6 +1552,20 @@
 :data:`unittest.defaultTestLoader`. Using a subclass or instance, however,
 allows customization of some configurable properties.
 
+ :class:`TestLoader` objects have the following attributes:
+
+
+ .. attribute:: errors
+
+ A list of the non-fatal errors encountered while loading tests. Not reset
+ by the loader at any point. Fatal errors are signalled by the relevant
+ a method raising an exception to the caller. Non-fatal errors are also
+ indicated by a synthetic test that will raise the original error when
+ run.
+
+ .. versionadded:: 3.5
+
+
 :class:`TestLoader` objects have the following methods:
 
 
diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py
--- a/Lib/unittest/loader.py
+++ b/Lib/unittest/loader.py
@@ -21,19 +21,22 @@
 
 
 def _make_failed_import_test(name, suiteClass):
- message = 'Failed to import test module: %s\n%s' % (name, traceback.format_exc())
+ message = 'Failed to import test module: %s\n%s' % (
+ name, traceback.format_exc())
 return _make_failed_test('ModuleImportFailure', name, ImportError(message),
- suiteClass)
+ suiteClass, message)
 
 def _make_failed_load_tests(name, exception, suiteClass):
- return _make_failed_test('LoadTestsFailure', name, exception, suiteClass)
+ message = 'Failed to call load_tests:\n%s' % (traceback.format_exc(),)
+ return _make_failed_test(
+ 'LoadTestsFailure', name, exception, suiteClass, message)
 
-def _make_failed_test(classname, methodname, exception, suiteClass):
+def _make_failed_test(classname, methodname, exception, suiteClass, message):
 def testFailure(self):
 raise exception
 attrs = {methodname: testFailure}
 TestClass = type(classname, (case.TestCase,), attrs)
- return suiteClass((TestClass(methodname),))
+ return suiteClass((TestClass(methodname),)), message
 
 def _make_skipped_test(methodname, exception, suiteClass):
 @case.skip(str(exception))
@@ -59,6 +62,10 @@
 suiteClass = suite.TestSuite
 _top_level_dir = None
 
+ def __init__(self):
+ super(TestLoader, self).__init__()
+ self.errors = []
+
 def loadTestsFromTestCase(self, testCaseClass):
 """Return a suite of all tests cases contained in testCaseClass"""
 if issubclass(testCaseClass, suite.TestSuite):
@@ -107,8 +114,10 @@
 try:
 return load_tests(self, tests, pattern)
 except Exception as e:
- return _make_failed_load_tests(module.__name__, e,
- self.suiteClass)
+ error_case, error_message = _make_failed_load_tests(
+ module.__name__, e, self.suiteClass)
+ self.errors.append(error_message)
+ return error_case
 return tests
 
 def loadTestsFromName(self, name, module=None):
@@ -336,7 +345,10 @@
 except case.SkipTest as e:
 yield _make_skipped_test(name, e, self.suiteClass)
 except:
- yield _make_failed_import_test(name, self.suiteClass)
+ error_case, error_message = \
+ _make_failed_import_test(name, self.suiteClass)
+ self.errors.append(error_message)
+ yield error_case
 else:
 mod_file = os.path.abspath(getattr(module, '__file__', full_path))
 realpath = _jython_aware_splitext(os.path.realpath(mod_file))
@@ -362,7 +374,10 @@
 except case.SkipTest as e:
 yield _make_skipped_test(name, e, self.suiteClass)
 except:
- yield _make_failed_import_test(name, self.suiteClass)
+ error_case, error_message = \
+ _make_failed_import_test(name, self.suiteClass)
+ self.errors.append(error_message)
+ yield error_case
 else:
 load_tests = getattr(package, 'load_tests', None)
 tests = self.loadTestsFromModule(package, pattern=pattern)
diff --git a/Lib/unittest/test/test_discovery.py b/Lib/unittest/test/test_discovery.py
--- a/Lib/unittest/test/test_discovery.py
+++ b/Lib/unittest/test/test_discovery.py
@@ -399,6 +399,13 @@
 suite = loader.discover('.')
 self.assertIn(os.getcwd(), sys.path)
 self.assertEqual(suite.countTestCases(), 1)
+ # Errors loading the suite are also captured for introspection.
+ self.assertNotEqual([], loader.errors)
+ self.assertEqual(1, len(loader.errors))
+ error = loader.errors[0]
+ self.assertTrue(
+ 'Failed to import test module: test_this_does_not_exist' in error,
+ 'missing error string in %r' % error)
 test = list(list(suite)[0])[0] # extract test from suite
 
 with self.assertRaises(ImportError):
@@ -418,6 +425,13 @@
 
 self.assertIn(abspath('/foo'), sys.path)
 self.assertEqual(suite.countTestCases(), 1)
+ # Errors loading the suite are also captured for introspection.
+ self.assertNotEqual([], loader.errors)
+ self.assertEqual(1, len(loader.errors))
+ error = loader.errors[0]
+ self.assertTrue(
+ 'Failed to import test module: my_package' in error,
+ 'missing error string in %r' % error)
 test = list(list(suite)[0])[0] # extract test from suite
 with self.assertRaises(ImportError):
 test.my_package()
diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py
--- a/Lib/unittest/test/test_loader.py
+++ b/Lib/unittest/test/test_loader.py
@@ -24,6 +24,13 @@
 
 class Test_TestLoader(unittest.TestCase):
 
+ ### Basic object tests
+ ################################################################
+
+ def test___init__(self):
+ loader = unittest.TestLoader()
+ self.assertEqual([], loader.errors)
+
 ### Tests for TestLoader.loadTestsFromTestCase
 ################################################################
 
@@ -336,6 +343,13 @@
 suite = loader.loadTestsFromModule(m)
 self.assertIsInstance(suite, unittest.TestSuite)
 self.assertEqual(suite.countTestCases(), 1)
+ # Errors loading the suite are also captured for introspection.
+ self.assertNotEqual([], loader.errors)
+ self.assertEqual(1, len(loader.errors))
+ error = loader.errors[0]
+ self.assertTrue(
+ 'Failed to call load_tests:' in error,
+ 'missing error string in %r' % error)
 test = list(suite)[0]
 
 self.assertRaisesRegex(TypeError, "some failure", test.m)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -186,6 +186,10 @@
 - Issue #9351: Defaults set with set_defaults on an argparse subparser
 are no longer ignored when also set on the parent parser.
 
+- Issue #19746: Make it possible to examine the errors from unittest
+ discovery without executing the test suite. The new `errors` attribute
+ on TestLoader exposes these non-fatal errors encountered during discovery.
+
 - Issue #21991: Make email.headerregistry's header 'params' attributes
 be read-only (MappingProxyType). Previously the dictionary was modifiable
 but a new one was created on each access of the attribute.
-- 
Repository URL: https://hg.python.org/cpython
