New regression testing harness.
(testall.out): obsolete. see individual test output files in the
Output directory.
(testall.py): now contains only the list of all tests
(autotest.py): The new testing harness. Supports -g option to
generate output files instead of verifying; -w option to warn about
non-tested optional modules; additional command line arguments to
generate or test individual tests (useful for generating output file
of new tests).
diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py
index 79d9b11..0d2fa8e 100644
--- a/Lib/test/autotest.py
+++ b/Lib/test/autotest.py
@@ -1,34 +1,58 @@
-# Automatic Python regression test.
-#
-# Some essential parts of the Python interpreter are tested by the module
-# 'testall'. (Despite its name, it doesn't test everything -- that would
-# be a truly Herculean task!) When a test fails, 'testall' raises an
-# exception. When all tests succeed, it produces quite a lot of output.
-#
-# For a normal regression test, this output is never looked at unless
-# something goes wrong. Thus, it would be wise to suppress the output
-# normally. This module does that, but it doesn't just throw the output
-# from 'testall' away -- it compares it with the output from a previous
-# run. If a difference is noticed it raises an exception; if all is well,
-# it prints nothing except 'All tests OK.' at the very end.
-#
-# The output from a previous run is supposed to be in a file 'testall.out'
-# somewhere on the search path for modules (sys.path, initialized from
-# $PYTHONPATH plus some default places).
-#
-# Of course, if the normal output of the tests is changed because the
-# tests have been changed (rather than a test producing the wrong output),
-# 'autotest' will fail as well. In this case, run 'testall' manually
-# and direct its output to 'testall.out'.
-#
-# The comparison uses (and demonstrates!) a rather new Python feature:
-# program output that normally goes to stdout (by 'print' statements
-# or by writing directly to sys.stdout) can be redirected to an
-# arbitrary class instance as long as it has a 'write' method.
+"""
+Automatic Python regression test.
+
+The list of individual tests is contained in the `testall' module.
+These test some (but not all) essential parts of the Python
+interpreter and built-in modules. When a test fails, an exception is
+raised and testing halts. When a test succeeds, it can produce quite
+a lot of output, which is compared against the output from a previous
+run. If a difference is noticed it raises an exception; if all is
+well, it prints nothing except 'All tests OK.' at the very end.
+
+The output from a previous run is supposed to be contained in separate
+files (one per test) in the `Output' subdirectory somewhere on the
+search path for modules (sys.path, initialized from $PYTHONPATH plus
+some default places).
+
+Of course, if the normal output of the tests is changed because the
+tests have been changed (rather than a test producing the wrong
+output), 'autotest' will fail as well. In this case, run 'autotest'
+with the -g option.
+
+Usage:
+
+ %s [-g] [-w] [-h] [test1 [test2 ...]]
+
+Options:
+
+ -g, --generate : generate the output files instead of verifying
+ the results
+
+ -w, --warn : warn about un-importable tests
+
+ -h, --help : print this message
+
+If individual tests are provided on the command line, only those tests
+will be performed or generated. Otherwise, all tests (as contained in
+testall.py) will be performed.
+
+"""
import os
import sys
+import getopt
+import traceback
+from test_support import *
+# Exception raised when the test failed (not the same as in test_support)
+TestFailed = 'autotest.TestFailed'
+
+# defaults
+generate = 0
+warn = 0
+
+
+
# Function to find a file somewhere on sys.path
def findfile(filename):
for dirname in sys.path:
@@ -37,9 +61,8 @@
return fullname
return filename # Will cause exception later
-# Exception raised when the test failed (not the same as in test_support)
-TestFailed = 'autotest.TestFailed'
+
# Class substituted for sys.stdout, to compare it with the given file
class Compare:
def __init__(self, filename):
@@ -50,18 +73,73 @@
raise TestFailed, \
'Writing: '+`data`+', expected: '+`expected`
def close(self):
+ leftover = self.fp.read()
+ if leftover:
+ raise TestFailed, 'Unread: '+`leftover`
self.fp.close()
+
# The main program
-def main():
- import sys
- filename = findfile('testall.out')
+def usage(status):
+ print __doc__ % sys.argv[0]
+ sys.exit(status)
+
+
+
+def do_one_test(t, outdir):
+ filename = os.path.join(outdir, t)
real_stdout = sys.stdout
try:
- sys.stdout = Compare(filename)
- import testall
+ if generate:
+ print 'Generating:', filename
+ sys.stdout = open(filename, 'w')
+ else:
+ sys.stdout = Compare(filename)
+ print t
+ unload(t)
+ try:
+ __import__(t, globals(), locals())
+ except ImportError, msg:
+ if warn:
+ sys.stderr.write(msg+': Un-installed'
+ ' optional module?\n')
finally:
+ sys.stdout.close()
sys.stdout = real_stdout
+
+
+
+def main():
+ global generate
+ global warn
+ try:
+ opts, args = getopt.getopt(
+ sys.argv[1:], 'ghw',
+ ['generate', 'help', 'warn'])
+ except getopt.error, msg:
+ print msg
+ usage(1)
+ for opt, val in opts:
+ if opt in ['-h', '--help']:
+ usage(0)
+ elif opt in ['-g', '--generate']:
+ generate = 1
+ elif opt in ['-w', '--warn']:
+ warn = 1
+
+ # find the output directory
+ outdir = findfile('Output')
+ if args:
+ tests = args
+ else:
+ import testall
+ tests = testall.tests
+ for test in tests:
+ try:
+ do_one_test(test, outdir)
+ except TestFailed, msg:
+ print 'Failure of test:', test
+ traceback.print_exc()
print 'All tests OK.'
main()
diff --git a/Lib/test/testall.out b/Lib/test/testall.out
deleted file mode 100644
index 412cba4..0000000
--- a/Lib/test/testall.out
+++ /dev/null
@@ -1,184 +0,0 @@
-test_grammar
-1. Parser
-1.1 Tokens
-1.1.1 Backslashes
-1.1.2 Numeric literals
-1.1.2.1 Plain integers
-1.1.2.2 Long integers
-1.1.2.3 Floating point
-1.1.3 String literals
-1.2 Grammar
-single_input
-file_input
-expr_input
-eval_input
-funcdef
-simple_stmt
-expr_stmt
-print_stmt
-1 2 3
-1 2 3
-1 1 1
-del_stmt
-pass_stmt
-flow_stmt
-break_stmt
-continue_stmt
-return_stmt
-raise_stmt
-import_stmt
-global_stmt
-exec_stmt
-if_stmt
-while_stmt
-for_stmt
-try_stmt
-suite
-test
-comparison
-binary mask ops
-shift ops
-additive ops
-multiplicative ops
-unary ops
-selectors
-atoms
-classdef
-test_opcodes
-2. Opcodes
-XXX Not yet fully implemented
-2.1 try inside for loop
-2.2 raise class exceptions
-test_operations
-3. Operations
-XXX Not yet implemented
-test_builtin
-4. Built-in functions
-test_b1
-__import__
-abs
-apply
-callable
-chr
-cmp
-coerce
-compile
-delattr
-dir
-divmod
-eval
-execfile
-filter
-float
-getattr
-hasattr
-hash
-hex
-id
-int
-len
-long
-map
-max
-min
-test_b2
-oct
-open
-ord
-pow
-range
-input and raw_input
-testing
-testing
-reduce
-reload
-repr
-round
-setattr
-str
-tuple
-type
-vars
-xrange
-test_exceptions
-5. Built-in exceptions
-AttributeError
-EOFError
-IOError
-ImportError
-IndexError
-KeyError
-KeyboardInterrupt
-(not testable in a script)
-MemoryError
-(not safe to test)
-NameError
-OverflowError
-RuntimeError
-(not used any more?)
-SyntaxError
-SystemError
-(hard to reproduce)
-SystemExit
-TypeError
-ValueError
-ZeroDivisionError
-test_types
-6. Built-in types
-6.1 Truth value testing
-6.2 Boolean operations
-6.3 Comparisons
-6.4 Numeric types (mostly conversions)
-6.4.1 32-bit integers
-6.4.2 Long integers
-6.4.3 Floating point numbers
-6.5 Sequence types
-6.5.1 Strings
-6.5.2 Tuples
-6.5.3 Lists
-6.5.3a Additional list operations
-6.6 Mappings == Dictionaries
-test_math
-math module, testing with eps 1e-05
-constants
-acos
-asin
-atan
-atan2
-ceil
-cos
-cosh
-exp
-fabs
-floor
-fmod
-frexp
-hypot
-ldexp
-log
-log10
-modf
-pow
-sin
-sinh
-sqrt
-tan
-tanh
-test_time
-test_array
-test_strop
-test_md5
-MD5 test suite:
-MD5 ("") = d41d8cd98f00b204e9800998ecf8427e
-MD5 ("a") = 0cc175b9c0f1b6a831c399e269772661
-MD5 ("abc") = 900150983cd24fb0d6963f7d28e17f72
-MD5 ("message digest") = f96b697d7cb7938d525a2f31aaf161d0
-MD5 ("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
-MD5 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
-MD5 ("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
-test_cmath
-test_crypt
-Test encryption: abeTm2pJKypwA
-test_dbm
-test_new
-Passed all tests.
diff --git a/Lib/test/testall.py b/Lib/test/testall.py
index b48accc..a132bfe 100644
--- a/Lib/test/testall.py
+++ b/Lib/test/testall.py
@@ -1,25 +1,19 @@
-# testall.py -- a regression test for the Python interpreter.
-# To run the tests, execute "import testall" in a clean interpreter.
-# It is a good idea to do this whenever you build a new interpreter.
-# Remember to add new tests when new features are added!
+# This file now contains only the list of separate regression tests.
+# All of the testing harness is now contained in autotest.py.
-import sys
-from test_support import *
-
-print 'test_grammar'
-forget('test_grammar')
-import test_grammar
-
-for t in ['test_opcodes', 'test_operations', 'test_builtin',
- 'test_exceptions', 'test_types', 'test_math', 'test_time',
- 'test_array', 'test_strop', 'test_md5', 'test_cmath',
- 'test_crypt', 'test_dbm', 'test_new',
- ]:
- print t
- unload(t)
- try:
- __import__(t, globals(), locals())
- except ImportError, msg:
- sys.stderr.write('%s. Uninstalled optional module?\n' % msg)
-
-print 'Passed all tests.'
+tests = ['test_grammar',
+ 'test_opcodes',
+ 'test_operations',
+ 'test_builtin',
+ 'test_exceptions',
+ 'test_types',
+ 'test_math',
+ 'test_time',
+ 'test_array',
+ 'test_strop',
+ 'test_md5',
+ 'test_cmath',
+ 'test_crypt',
+ 'test_dbm',
+ 'test_new',
+ ]