csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | # Copyright 2016 Google Inc. |
| 4 | # |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | from __future__ import print_function |
| 9 | from _benchresult import BenchResult |
| 10 | from argparse import ArgumentParser |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 11 | from collections import defaultdict, namedtuple |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 12 | from datetime import datetime |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 13 | import operator |
| 14 | import os |
| 15 | import sys |
| 16 | import tempfile |
| 17 | import urllib |
| 18 | import urlparse |
| 19 | import webbrowser |
| 20 | |
csmartdalton | bf41fa8 | 2016-09-23 11:36:11 -0700 | [diff] [blame] | 21 | __argparse = ArgumentParser(description=""" |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 22 | |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 23 | Formats skpbench.py outputs as csv. |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 24 | |
| 25 | This script can also be used to generate a Google sheet: |
| 26 | |
| 27 | (1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: |
| 28 | https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegbaiigmenfmjfclcdgdpimamgkj |
| 29 | |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 30 | (2) Update your global OS file associations to use Chrome for .csv files. |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 31 | |
| 32 | (3) Run parseskpbench.py with the --open flag. |
| 33 | |
csmartdalton | bf41fa8 | 2016-09-23 11:36:11 -0700 | [diff] [blame] | 34 | """) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 35 | |
| 36 | __argparse.add_argument('-r', '--result', |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 37 | choices=['accum', 'median', 'max', 'min'], default='accum', |
| 38 | help="result to use for cell values") |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 39 | __argparse.add_argument('-f', '--force', |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 40 | action='store_true', help='silently ignore warnings') |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 41 | __argparse.add_argument('-o', '--open', |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 42 | action='store_true', |
| 43 | help="generate a temp file and open it (theoretically in a web browser)") |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 44 | __argparse.add_argument('-n', '--name', |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 45 | default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), |
| 46 | help="if using --open, a name for the temp file") |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 47 | __argparse.add_argument('sources', |
Kevin Lubick | 9c2249f | 2016-11-10 14:19:00 -0500 | [diff] [blame] | 48 | nargs='+', help="source files that contain skpbench results ('-' for stdin)") |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 49 | |
| 50 | FLAGS = __argparse.parse_args() |
| 51 | |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 52 | RESULT_QUALIFIERS = ('sample_ms', 'clock', 'metric') |
| 53 | |
| 54 | class FullConfig(namedtuple('fullconfig', ('config',) + RESULT_QUALIFIERS)): |
| 55 | def qualified_name(self, qualifiers=RESULT_QUALIFIERS): |
| 56 | return get_qualified_name(self.config.replace(',', ' '), |
| 57 | {x:getattr(self, x) for x in qualifiers}) |
| 58 | |
| 59 | def get_qualified_name(name, qualifiers): |
| 60 | if not qualifiers: |
| 61 | return name |
| 62 | else: |
| 63 | args = ('%s=%s' % (k,v) for k,v in qualifiers.iteritems()) |
| 64 | return '%s (%s)' % (name, ' '.join(args)) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 65 | |
| 66 | class Parser: |
| 67 | def __init__(self): |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 68 | self.sheet_qualifiers = {x:None for x in RESULT_QUALIFIERS} |
| 69 | self.config_qualifiers = set() |
| 70 | self.fullconfigs = list() # use list to preserve the order. |
| 71 | self.rows = defaultdict(dict) |
| 72 | self.cols = defaultdict(dict) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 73 | |
| 74 | def parse_file(self, infile): |
| 75 | for line in infile: |
| 76 | match = BenchResult.match(line) |
| 77 | if not match: |
| 78 | continue |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 79 | |
| 80 | fullconfig = FullConfig(*(match.get_string(x) |
| 81 | for x in FullConfig._fields)) |
| 82 | if not fullconfig in self.fullconfigs: |
| 83 | self.fullconfigs.append(fullconfig) |
| 84 | |
| 85 | for qualifier, value in self.sheet_qualifiers.items(): |
| 86 | if value is None: |
| 87 | self.sheet_qualifiers[qualifier] = match.get_string(qualifier) |
| 88 | elif value != match.get_string(qualifier): |
| 89 | del self.sheet_qualifiers[qualifier] |
| 90 | self.config_qualifiers.add(qualifier) |
| 91 | |
| 92 | self.rows[match.bench][fullconfig] = match.get_string(FLAGS.result) |
| 93 | self.cols[fullconfig][match.bench] = getattr(match, FLAGS.result) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 94 | |
| 95 | def print_csv(self, outfile=sys.stdout): |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 96 | # Write the title. |
| 97 | print(get_qualified_name(FLAGS.result, self.sheet_qualifiers), file=outfile) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 98 | |
| 99 | # Write the header. |
| 100 | outfile.write('bench,') |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 101 | for fullconfig in self.fullconfigs: |
| 102 | outfile.write('%s,' % fullconfig.qualified_name(self.config_qualifiers)) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 103 | outfile.write('\n') |
| 104 | |
| 105 | # Write the rows. |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 106 | for bench, row in self.rows.iteritems(): |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 107 | outfile.write('%s,' % bench) |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 108 | for fullconfig in self.fullconfigs: |
| 109 | if fullconfig in row: |
| 110 | outfile.write('%s,' % row[fullconfig]) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 111 | elif FLAGS.force: |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 112 | outfile.write('NULL,') |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 113 | else: |
csmartdalton | d7a9db6 | 2016-09-22 05:10:02 -0700 | [diff] [blame] | 114 | raise ValueError("%s: missing value for %s. (use --force to ignore)" % |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 115 | (bench, |
| 116 | fullconfig.qualified_name(self.config_qualifiers))) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 117 | outfile.write('\n') |
| 118 | |
| 119 | # Add simple, literal averages. |
| 120 | if len(self.rows) > 1: |
| 121 | outfile.write('\n') |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 122 | self._print_computed_row('MEAN', |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 123 | lambda col: reduce(operator.add, col.values()) / len(col), |
| 124 | outfile=outfile) |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 125 | self._print_computed_row('GEOMEAN', |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 126 | lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), |
| 127 | outfile=outfile) |
| 128 | |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 129 | def _print_computed_row(self, name, func, outfile=sys.stdout): |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 130 | outfile.write('%s,' % name) |
csmartdalton | c6618dd | 2016-10-05 08:42:03 -0700 | [diff] [blame] | 131 | for fullconfig in self.fullconfigs: |
| 132 | if len(self.cols[fullconfig]) != len(self.rows): |
| 133 | outfile.write('NULL,') |
| 134 | continue |
| 135 | outfile.write('%.4g,' % func(self.cols[fullconfig])) |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 136 | outfile.write('\n') |
| 137 | |
csmartdalton | 4b5179b | 2016-09-19 11:03:58 -0700 | [diff] [blame] | 138 | def main(): |
| 139 | parser = Parser() |
| 140 | |
| 141 | # Parse the input files. |
| 142 | for src in FLAGS.sources: |
| 143 | if src == '-': |
| 144 | parser.parse_file(sys.stdin) |
| 145 | else: |
| 146 | with open(src, mode='r') as infile: |
| 147 | parser.parse_file(infile) |
| 148 | |
| 149 | # Print the csv. |
| 150 | if not FLAGS.open: |
| 151 | parser.print_csv() |
| 152 | else: |
| 153 | dirname = tempfile.mkdtemp() |
| 154 | basename = FLAGS.name |
| 155 | if os.path.splitext(basename)[1] != '.csv': |
| 156 | basename += '.csv'; |
| 157 | pathname = os.path.join(dirname, basename) |
| 158 | with open(pathname, mode='w') as tmpfile: |
| 159 | parser.print_csv(outfile=tmpfile) |
| 160 | fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) |
| 161 | print('opening %s' % fileuri) |
| 162 | webbrowser.open(fileuri) |
| 163 | |
| 164 | |
| 165 | if __name__ == '__main__': |
| 166 | main() |