Note that a number of the changes listed below were not applicable to the Py3k branch, and hence the corresponding
files are unchanged in this checkin. This checkin is also the first time the environment checking in regrtest has
been forward ported to the Py3k branch.
This checkin causes test_xmlrpc to fail - see issue 7165 (it's a bug in the 3.x version of xmlrpc.server)
I am also getting a failure in test_telnetlib, but it isn't clear yet if that is due to these changes.
Recorded merge of revisions 75400-75401,75404,75406,75414,75416,75422,75425-75428,75435,75439,75441-75444,75447-75449,75451-75453,75455-75458,75460-75469,75471-75473,75475-75477,75479-75481,75483,75486-75489 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r75400 | r.david.murray | 2009-10-14 23:58:07 +1000 (Wed, 14 Oct 2009) | 6 lines
Enhanced Issue 7058 patch, which will not be backported. Refactors the
code, adds checks for stdin/out/err, cwd, and sys.path, and adds a new
section in the summary for tests that modify the environment (thanks to
Ezio Melotti for that suggestion).
........
r75453 | nick.coghlan | 2009-10-17 16:33:05 +1000 (Sat, 17 Oct 2009) | 1 line
Correctly restore sys.stdout in test_descr
........
r75456 | nick.coghlan | 2009-10-17 17:30:40 +1000 (Sat, 17 Oct 2009) | 1 line
Enhancement to the new environment checking code to print the changed items under -vv. Also includes a small tweak to allow underscores in the names of resources.
........
r75457 | nick.coghlan | 2009-10-17 17:34:27 +1000 (Sat, 17 Oct 2009) | 1 line
Formatting tweak so that before and after values are vertically aligned
........
r75458 | nick.coghlan | 2009-10-17 18:21:21 +1000 (Sat, 17 Oct 2009) | 1 line
Check and revert expected sys.path alterations
........
r75461 | nick.coghlan | 2009-10-18 00:40:54 +1000 (Sun, 18 Oct 2009) | 1 line
Restore original sys.path when running TTK tests
........
r75462 | nick.coghlan | 2009-10-18 01:09:41 +1000 (Sun, 18 Oct 2009) | 1 line
Don't invoke reload(sys) and use StringIO objects instead of real files to capture stdin and stdout when needed (ensures all sys attributes remain unmodified after test_xmlrpc runs)
........
r75463 | nick.coghlan | 2009-10-18 01:23:08 +1000 (Sun, 18 Oct 2009) | 1 line
Revert changes made to environment in test_httpservers
........
r75465 | nick.coghlan | 2009-10-18 01:45:52 +1000 (Sun, 18 Oct 2009) | 1 line
Move restoration of the os.environ object into the context manager where it belongs
........
r75466 | nick.coghlan | 2009-10-18 01:48:16 +1000 (Sun, 18 Oct 2009) | 1 line
Also check and restore identity of sys.path, sys.argv and os.environ rather than just their values (this picked up a few more misbehaving tests)
........
r75467 | nick.coghlan | 2009-10-18 01:57:42 +1000 (Sun, 18 Oct 2009) | 1 line
Avoid replacing existing modules and sys.path in import tests
........
r75468 | nick.coghlan | 2009-10-18 02:19:51 +1000 (Sun, 18 Oct 2009) | 1 line
Don't replace sys.path in test_site
........
r75481 | nick.coghlan | 2009-10-18 15:38:48 +1000 (Sun, 18 Oct 2009) | 1 line
Using CleanImport to revert a reload of the os module doesn't work due to function registrations in copy_reg. The perils of reloading modules even for tests...
........
r75486 | nick.coghlan | 2009-10-18 20:29:10 +1000 (Sun, 18 Oct 2009) | 1 line
Silence a deprecation warning by using the appropriate replacement construct
........
r75489 | nick.coghlan | 2009-10-18 20:56:21 +1000 (Sun, 18 Oct 2009) | 1 line
Restore sys.path in test_tk
........
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index a544d67..e808581 100755
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -353,6 +353,7 @@
bad = []
skipped = []
resource_denieds = []
+ environment_changed = []
if findleaks:
try:
@@ -429,11 +430,13 @@
test_times.append((test_time, test))
if ok > 0:
good.append(test)
- elif ok == 0:
+ elif -2 < ok <= 0:
bad.append(test)
+ if ok == -1:
+ environment_changed.append(test)
else:
skipped.append(test)
- if ok == -2:
+ if ok == -3:
resource_denieds.append(test)
if use_mp:
@@ -539,6 +542,7 @@
good.sort()
bad.sort()
skipped.sort()
+ environment_changed.sort()
if good and not quiet:
if not bad and not skipped and len(good) > 1:
@@ -553,8 +557,14 @@
for time, test in test_times[:10]:
print("%s: %.1fs" % (test, time))
if bad:
- print(count(len(bad), "test"), "failed:")
- printlist(bad)
+ bad = sorted(set(bad) - set(environment_changed))
+ if bad:
+ print(count(len(bad), "test"), "failed:")
+ printlist(bad)
+ if environment_changed:
+ print("{} altered the execution environment:".format(
+ count(len(environment_changed), "test")))
+ printlist(environment_changed)
if skipped and not quiet:
print(count(len(skipped), "test"), "skipped:")
printlist(skipped)
@@ -657,8 +667,10 @@
debug -- if true, print tracebacks for failed tests regardless of
verbose setting
Return:
- -2 test skipped because resource denied
- -1 test skipped for some other reason
+ -4 KeyboardInterrupt when run under -j
+ -3 test skipped because resource denied
+ -2 test skipped for some other reason
+ -1 test failed because it changed the execution environment
0 test failed
1 test passed
"""
@@ -672,6 +684,118 @@
finally:
cleanup_test_droppings(test, verbose)
+# Unit tests are supposed to leave the execution environment unchanged
+# once they complete. But sometimes tests have bugs, especially when
+# tests fail, and the changes to environment go on to mess up other
+# tests. This can cause issues with buildbot stability, since tests
+# are run in random order and so problems may appear to come and go.
+# There are a few things we can save and restore to mitigate this, and
+# the following context manager handles this task.
+
+class saved_test_environment:
+ """Save bits of the test environment and restore them at block exit.
+
+ with saved_test_environment(testname, verbose, quiet):
+ #stuff
+
+ Unless quiet is True, a warning is printed to stderr if any of
+ the saved items was changed by the test. The attribute 'changed'
+ is initially False, but is set to True if a change is detected.
+
+ If verbose is more than 1, the before and after state of changed
+ items is also printed.
+ """
+
+ changed = False
+
+ def __init__(self, testname, verbose=0, quiet=False):
+ self.testname = testname
+ self.verbose = verbose
+ self.quiet = quiet
+
+ # To add things to save and restore, add a name XXX to the resources list
+ # and add corresponding get_XXX/restore_XXX functions. get_XXX should
+ # return the value to be saved and compared against a second call to the
+ # get function when test execution completes. restore_XXX should accept
+ # the saved value and restore the resource using it. It will be called if
+ # and only if a change in the value is detected.
+ #
+ # Note: XXX will have any '.' replaced with '_' characters when determining
+ # the corresponding method names.
+
+ resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr',
+ 'os.environ', 'sys.path')
+
+ def get_sys_argv(self):
+ return id(sys.argv), sys.argv, sys.argv[:]
+ def restore_sys_argv(self, saved_argv):
+ sys.argv = saved_argv[1]
+ sys.argv[:] = saved_argv[2]
+
+ def get_cwd(self):
+ return os.getcwd()
+ def restore_cwd(self, saved_cwd):
+ os.chdir(saved_cwd)
+
+ def get_sys_stdout(self):
+ return sys.stdout
+ def restore_sys_stdout(self, saved_stdout):
+ sys.stdout = saved_stdout
+
+ def get_sys_stderr(self):
+ return sys.stderr
+ def restore_sys_stderr(self, saved_stderr):
+ sys.stderr = saved_stderr
+
+ def get_sys_stdin(self):
+ return sys.stdin
+ def restore_sys_stdin(self, saved_stdin):
+ sys.stdin = saved_stdin
+
+ def get_os_environ(self):
+ return id(os.environ), os.environ, dict(os.environ)
+ def restore_os_environ(self, saved_environ):
+ os.environ = saved_environ[1]
+ os.environ.clear()
+ os.environ.update(saved_environ[2])
+
+ def get_sys_path(self):
+ return id(sys.path), sys.path, sys.path[:]
+ def restore_sys_path(self, saved_path):
+ sys.path = saved_path[1]
+ sys.path[:] = saved_path[2]
+
+ def resource_info(self):
+ for name in self.resources:
+ method_suffix = name.replace('.', '_')
+ get_name = 'get_' + method_suffix
+ restore_name = 'restore_' + method_suffix
+ yield name, getattr(self, get_name), getattr(self, restore_name)
+
+ def __enter__(self):
+ self.saved_values = dict((name, get()) for name, get, restore
+ in self.resource_info())
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ for name, get, restore in self.resource_info():
+ current = get()
+ original = self.saved_values[name]
+ # Check for changes to the resource's value
+ if current != original:
+ self.changed = True
+ restore(original)
+ if not self.quiet:
+ print("Warning -- {} was modified by {}".format(
+ name, self.testname),
+ file=sys.stderr)
+ if self.verbose > 1:
+ print(" Before: {}\n After: {} ".format(
+ original, current),
+ file=sys.stderr)
+ return False
+
+
def runtest_inner(test, verbose, quiet,
testdir=None, huntrleaks=False, debug=False):
support.unload(test)
@@ -692,18 +816,20 @@
else:
# Always import it from the test package
abstest = 'test.' + test
- start_time = time.time()
- the_package = __import__(abstest, globals(), locals(), [])
- the_module = getattr(the_package, test)
- # Old tests run to completion simply as a side-effect of
- # being imported. For tests based on unittest or doctest,
- # explicitly invoke their test_main() function (if it exists).
- indirect_test = getattr(the_module, "test_main", None)
- if indirect_test is not None:
- indirect_test()
- if huntrleaks:
- refleak = dash_R(the_module, test, indirect_test, huntrleaks)
- test_time = time.time() - start_time
+ with saved_test_environment(test, verbose, quiet) as environment:
+ start_time = time.time()
+ the_package = __import__(abstest, globals(), locals(), [])
+ the_module = getattr(the_package, test)
+ # Old tests run to completion simply as a side-effect of
+ # being imported. For tests based on unittest or doctest,
+ # explicitly invoke their test_main() function (if it exists).
+ indirect_test = getattr(the_module, "test_main", None)
+ if indirect_test is not None:
+ indirect_test()
+ if huntrleaks:
+ refleak = dash_R(the_module, test, indirect_test,
+ huntrleaks)
+ test_time = time.time() - start_time
finally:
sys.stdout = save_stdout
# Restore what we saved if needed, but also complain if the test
@@ -721,12 +847,12 @@
if not quiet:
print(test, "skipped --", msg)
sys.stdout.flush()
- return -2, test_time
+ return -3, test_time
except unittest.SkipTest as msg:
if not quiet:
print(test, "skipped --", msg)
sys.stdout.flush()
- return -1, test_time
+ return -2, test_time
except KeyboardInterrupt:
raise
except support.TestFailed as msg:
@@ -744,6 +870,8 @@
else:
if refleak:
return 0, test_time
+ if environment.changed:
+ return -1, test_time
return 1, test_time
def cleanup_test_droppings(testname, verbose):