Tools/pybench/pybench.py

#!/usr/local/bin/python -O

""" A Python Benchmark Suite

"""
#
# Note: Please keep this module compatible to Python 1.5.2.
#
# Tests may include features in later Python versions, but these
# should then be embedded in try-except clauses in the configuration
# module Setup.py.
#

# pybench Copyright
__copyright__ = """\
Copyright (c), 1997-2006, Marc-Andre Lemburg (mal@lemburg.com)
Copyright (c), 2000-2006, eGenix.com Software GmbH (info@egenix.com)

                   All Rights Reserved.

Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee or royalty is hereby
granted, provided that the above copyright notice appear in all copies
and that both that copyright notice and this permission notice appear
in supporting documentation or portions thereof, including
modifications, that you make.

THE AUTHOR MARC-ANDRE LEMBURG DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
WITH THE USE OR PERFORMANCE OF THIS SOFTWARE !
"""

# Version number
__version__ = '1.3'

#
# NOTE: Use xrange for all test loops unless you want to face
#       a 20MB process !
#
# All tests should have rounds set to values so that a run()
# takes between 20-50 seconds. This is to get fairly good
# clock() values. You can use option -w to speedup the tests
# by a fixed integer factor (the "warp factor").
#

import sys,time,operator
from CommandLine import *

try:
    import cPickle
    pickle = cPickle
except ImportError:
    import pickle

### Test baseclass

class Test:

    """ All test must have this class as baseclass. It provides
        the necessary interface to the benchmark machinery.

        The tests must set .rounds to a value high enough to let the
        test run between 20-50 seconds. This is needed because
        clock()-timing only gives rather inaccurate values (on Linux,
        for example, it is accurate to a few hundreths of a
        second). If you don't want to wait that long, use a warp
        factor larger than 1.

        It is also important to set the .operations variable to a
        value representing the number of "virtual operations" done per
        call of .run().

        If you change a test in some way, don't forget to increase
        it's version number.

    """

    ### Instance variables that each test should override

    # Version number of the test as float (x.yy); this is important
    # for comparisons of benchmark runs - tests with unequal version
    # number will not get compared.
    version = 1.0

    # The number of abstract operations done in each round of the
    # test. An operation is the basic unit of what you want to
    # measure. The benchmark will output the amount of run-time per
    # operation. Note that in order to raise the measured timings
    # significantly above noise level, it is often required to repeat
    # sets of operations more than once per test round. The measured
    # overhead per test round should be less than 1 second.
    operations = 1

    # Number of rounds to execute per test run. This should be
    # adjusted to a figure that results in a test run-time of between
    # 20-50 seconds.
    rounds = 10000

    ### Internal variables

    # Mark this class as implementing a test
    is_a_test = 1

    # Misc. internal variables
    last_timing = (0,0,0) # last timing (real,run,calibration)
    warp = 1            # warp factor this test uses
    cruns = 20          # number of calibration runs
    overhead = None     # list of calibration timings

    def __init__(self,warp=1):

        if warp > 1:
            self.rounds = self.rounds / warp
            if self.rounds == 0:
                self.rounds = 1
            self.warp = warp
        self.times = []
        self.overhead = []
        # We want these to be in the instance dict, so that pickle
        # saves them
        self.version = self.version
        self.operations = self.operations
        self.rounds = self.rounds

    def run(self, cruns):

        """ Run the test in two phases: first calibrate, then
            do the actual test. Be careful to keep the calibration
            timing low w/r to the test timing.

        """
        test = self.test
        calibrate = self.calibrate
        clock = time.clock
        # first calibrate
        t = clock()
        calibrate()
        offset = clock() - t
        if cruns:
            for i in range(cruns-1):
                t = clock()
                calibrate()
                t = clock() - t
                if t < offset:
                    offset = t
        # now the real thing
        t = clock()
        test()
        t = clock() - t
        if t < 0.01:
            sys.exit("Lower warp required: test times < 10 ms are unreliable")
        self.last_timing = (t-offset,t,offset)
        self.times.append(t-offset)

    def calibrate(self):

        """ Calibrate the test.

            This method should execute everything that is needed to
            setup and run the test - except for the actual operations
            that you intend to measure. pybench uses this method to
            measure the test implementation overhead.

        """
        return

    def test(self):

        """ Run the test.

            The test needs to run self.rounds executing
            self.operations number of operations each.

        """
        # do some tests
        return

    def stat(self):

        """ Returns four values:
        minimum round time
        average time per round
        average time per operation
        average overhead time

        XXX Should this take warp factors into account?
        """
        runs = len(self.times)
        if runs == 0:
            return 0,0
        mintime = min(self.times)
        totaltime = reduce(operator.add,self.times,0.0)
        avg = totaltime / float(runs)
        op_avg = totaltime / float(runs * self.rounds * self.operations)
        if self.overhead:
            totaloverhead = reduce(operator.add,self.overhead,0.0)
            ov_avg = totaloverhead / float(runs)
        else:
            # use self.last_timing - not too accurate
            ov_avg = self.last_timing[2]
        return mintime, avg, op_avg, ov_avg

### Load Setup

# This has to be done after the definition of the Test class, since
# the Setup module will import subclasses using this class.

import Setup

### Benchmark base class

class Benchmark:

    name = '?'                  # Name of the benchmark
    rounds = 1                  # Number of rounds to run
    warp = 1                    # Warp factor
    roundtime = 0               # Average round time
    version = None              # Benchmark version number (see __init__)
                                # as float x.yy

    def __init__(self):

        self.tests = {}
        self.version = 0.31

    def load_tests(self, setupmod, warp=1, limitnames="", verbose=0):

        self.warp = warp
        if limitnames:
            limitnames = re.compile(limitnames, re.I)
        else:
            limitnames = None
        tests = self.tests
        if verbose:
            print 'Searching for tests ...',
        setupmod.__dict__.values()
        for c in setupmod.__dict__.values():
            if not hasattr(c,'is_a_test'):
                continue
            name = c.__name__
            if  name == 'Test':
                continue
            if limitnames is not None and limitnames.search(name) is None:
                continue
            tests[name] = c(warp)
        l = tests.keys()
        l.sort()
        if verbose:
            print
            for t in l:
                print '  ', t
            print len(l), "tests found"
        print

    def run(self, verbose, cruns):

        tests = self.tests.items()
        tests.sort()
        clock = time.clock
        print 'Running %i round(s) of the suite at warp factor %i:' % (self.rounds, self.warp)
        print
        roundtime = clock()
        for i in range(self.rounds):
            roundstarttime = clock()
            if verbose:
                print ' Round %-25i  real   abs    overhead' % (i+1)
            for j in range(len(tests)):
                name, t = tests[j]
                if verbose:
                    print '%30s:' % name,
                t.run(cruns)
                if verbose:
                    print '  %.3fr %.3fa %.3fo' % t.last_timing
            if verbose:
                print '                                 ----------------------'
                print '            Average round time:      %.3f seconds' % \
                    ((clock() - roundtime)/(i+1))
                print
            else:
                print '%d done in %.3f seconds' % (i+1, (clock() - roundstarttime))
        self.roundtime = (clock() - roundtime) / self.rounds
        print

    def print_stat(self, compare_to=None, hidenoise=0):

        if not compare_to:
            print '%-30s       min run    avg run    per oprn  overhead' % 'Tests:'
            print '-'*77
            tests = self.tests.items()
            tests.sort()
            totalmintime = 0
            for name,t in tests:
                mintime,avg,op_avg,ov_avg = t.stat()
                totalmintime += mintime
                print '%30s: %9.2f ms %9.2f ms %6.2f us  %6.2f' % \
                      (name,mintime*1000.0,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0)
            print '-'*77
            print '%30s: %9.2f ms' % \
                  ('Notional minimum round time', totalmintime * 1000.0)

        else:
            print 'Comparing with: %s (rounds=%i, warp=%i)' % \
                  (compare_to.name,compare_to.rounds,compare_to.warp)
            print '%-30s      min run     cmp run     avg run      diff' % \
                  'Tests:'
            print '-'*77
            tests = self.tests.items()
            tests.sort()
            compatible = 1
            totalmintime = other_totalmintime = 0
            for name, t in tests:
                mintime, avg, op_avg, ov_avg = t.stat()
                totalmintime += mintime
                try:
                    other = compare_to.tests[name]
                except KeyError:
                    other = None
                if other and other.version == t.version and \
                   other.operations == t.operations:
                    mintime1, avg1, op_avg1, ov_avg1 = other.stat()
                    other_totalmintime += mintime1
                    diff = ((mintime*self.warp)/(mintime1*other.warp) - 1.0)*100.0
                    if hidenoise and abs(qop_avg) < 10:
                        diff = ''
                    else:
                        diff = '%+7.2f%%' % diff
                else:
                    qavg, diff = 'n/a', 'n/a'
                    compatible = 0
                print '%30s: %8.2f ms %8.2f ms %8.2f ms  %8s' % \
                      (name,mintime*1000.0,mintime1*1000.0 * compare_to.warp/self.warp, avg*1000.0,diff)
            print '-'*77
            #
            # Summarise test results
            #
            if compatible and compare_to.roundtime > 0 and \
               compare_to.version == self.version:
                print '%30s: %8.2f ms %8.2f ms              %+7.2f%%' % \
                      ('Notional minimum round time', totalmintime * 1000.0,
                      other_totalmintime * 1000.0 * compare_to.warp/self.warp,
                       ((totalmintime*self.warp)/
                        (other_totalmintime*compare_to.warp)-1.0)*100.0)
            else:
                print '%30s: %9.2f ms                    n/a' % \
                      ('Notional minimum round time', totalmintime * 1000.0)
        print

def print_machine():

    import platform
    print 'Machine Details:'
    print '   Platform ID:  %s' % platform.platform()
    print '   Executable:   %s' % sys.executable
    # There's a bug in Python 2.2b1+...
    if sys.version[:6] == '2.2b1+':
        return
    print '   Python:       %s' % platform.python_version()
    print '   Compiler:     %s' % platform.python_compiler()
    buildno, builddate = platform.python_build()
    print '   Build:        %s (#%s)' % (builddate, buildno)

class PyBenchCmdline(Application):

    header = ("PYBENCH - a benchmark test suite for Python "
              "interpreters/compilers.")

    version = __version__

    options = [ArgumentOption('-n','number of rounds',Setup.Number_of_rounds),
               ArgumentOption('-f','save benchmark to file arg',''),
               ArgumentOption('-c','compare benchmark with the one in file arg',''),
               ArgumentOption('-s','show benchmark in file arg, then exit',''),
               SwitchOption('-S','show statistics of benchmarks',0),
               ArgumentOption('-w','set warp factor to arg',Setup.Warp_factor),
               SwitchOption('-d','hide noise in compares', 0),
               SwitchOption('-v','verbose output (not recommended)', 0),
               SwitchOption('--no-gc','disable garbage collection', 0),
               SwitchOption('--no-syscheck',
                    '"disable" sys check interval (set to sys.maxint)', 0),
               ArgumentOption('-t', 'tests containing substring', ''),
               ArgumentOption('-C', 'number of calibration runs', 20)
               ]

    about = """\
The normal operation is to run the suite and display the
results. Use -f to save them for later reuse or comparisms.

Examples:

python1.5 pybench.py -w 100 -f p15
python1.4 pybench.py -w 100 -f p14
python pybench.py -s p15 -c p14
"""
    copyright = __copyright__

    def handle_S(self, value):

        """ Display one line stats for each benchmark file given on the
            command line.

        """
        for benchmark in self.files:
            try:
                f = open(benchmark, 'rb')
                bench = pickle.load(f)
                f.close()
            except IOError:
                print '* Error opening/reading file %s' % repr(benchmark)
            else:
                print '%s,%-.2f,ms' % (benchmark, bench.roundtime*1000.0)
        return 0

    def main(self):

        rounds = self.values['-n']
        reportfile = self.values['-f']
        show_bench = self.values['-s']
        compare_to = self.values['-c']
        hidenoise = self.values['-d']
        warp = self.values['-w']
        nogc = self.values['--no-gc']
        limitnames = self.values['-t']
        verbose = self.verbose
        nosyscheck = self.values['--no-syscheck']
        cruns = self.values['-C']
        print "CRUNS:", cruns

        print 'PYBENCH',__version__

        # Switch off GC
        if nogc:
            try:
                import gc
            except ImportError:
                nogc = 0
            else:
                if self.values['--no-gc']:
                    gc.disable()
                    print 'NO GC'

        # maximise sys check interval
        if nosyscheck:
            sys.setcheckinterval(sys.maxint)
            print 'CHECKINTERVAL =', sys.maxint

        print

        if not compare_to:
            print_machine()
            print

        if compare_to:
            try:
                f = open(compare_to,'rb')
                bench = pickle.load(f)
                bench.name = compare_to
                f.close()
                compare_to = bench
            except IOError:
                print '* Error opening/reading file',compare_to
                compare_to = None

        if show_bench:
            try:
                f = open(show_bench,'rb')
                bench = pickle.load(f)
                bench.name = show_bench
                f.close()
                print 'Benchmark: %s (rounds=%i, warp=%i)' % \
                      (bench.name,bench.rounds,bench.warp)
                print
                bench.print_stat(compare_to, hidenoise)
            except IOError:
                print '* Error opening/reading file',show_bench
                print
            return

        if reportfile:
            if nogc:
                print 'Benchmark: %s (rounds=%i, warp=%i, no GC)' % \
                      (reportfile,rounds,warp)
            else:
                print 'Benchmark: %s (rounds=%i, warp=%i)' % \
                      (reportfile,rounds,warp)
            print

        # Create benchmark object
        bench = Benchmark()
        bench.rounds = rounds
        bench.load_tests(Setup, warp, limitnames, verbose)
        try:
            bench.run(verbose, cruns)
        except KeyboardInterrupt:
            print
            print '*** KeyboardInterrupt -- Aborting'
            print
            return
        bench.print_stat(compare_to)
        # ring bell
        sys.stderr.write('\007')

        if reportfile:
            try:
                f = open(reportfile,'wb')
                bench.name = reportfile
                pickle.dump(bench,f)
                f.close()
            except IOError:
                print '* Error opening/writing reportfile'

if __name__ == '__main__':
    PyBenchCmdline()