Martin Hořeňovský | 3b7511e | 2017-01-14 21:55:37 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | import subprocess, os, sys |
| 4 | import xml.etree.ElementTree as ET |
| 5 | from collections import defaultdict |
| 6 | from statistics import median, stdev |
| 7 | from datetime import datetime |
| 8 | |
| 9 | def get_commit_hash(): |
| 10 | res = subprocess.run('git rev-parse HEAD'.split(), check=True, stdout=subprocess.PIPE, universal_newlines=True) |
| 11 | return res.stdout.strip() |
| 12 | |
| 13 | if len(sys.argv) < 2: |
| 14 | print('Usage: {} benchmark-binary'.format(sys.argv[0])) |
| 15 | exit(1) |
| 16 | |
| 17 | |
| 18 | num_runs = 10 |
| 19 | data = defaultdict(list) |
| 20 | |
| 21 | |
| 22 | def parse_file(file): |
| 23 | |
| 24 | def recursive_search(node): |
| 25 | if node.tag == 'TestCase': |
| 26 | results = node.find('OverallResult') |
| 27 | time = results.get('durationInSeconds') |
| 28 | data[node.get('name')].append(float(time)) |
| 29 | elif node.tag in ('Group', 'Catch'): |
| 30 | for child in node: |
| 31 | recursive_search(child) |
| 32 | |
| 33 | tree = ET.parse(file) |
| 34 | recursive_search(tree.getroot()) |
| 35 | |
| 36 | def run_benchmarks(binary): |
| 37 | call = [binary] + '-d yes -r xml -o'.split() |
| 38 | for i in range(num_runs): |
| 39 | file = 'temp{}.xml'.format(i) |
| 40 | print('Run number {}'.format(i)) |
| 41 | subprocess.run(call + [file]) |
| 42 | parse_file(file) |
| 43 | # Remove file right after parsing, because benchmark output can be big |
| 44 | os.remove(file) |
| 45 | |
| 46 | |
| 47 | # Run benchmarks |
| 48 | run_benchmarks(sys.argv[1]) |
| 49 | |
| 50 | result_file = '{:%Y-%m-%dT%H-%M-%S}-{}.result'.format(datetime.now(), get_commit_hash()) |
| 51 | |
| 52 | |
| 53 | print('Writing results to {}'.format(result_file)) |
| 54 | with open(result_file, 'w') as file: |
| 55 | for k in sorted(data): |
| 56 | file.write('{}: median: {} (s), stddev: {} (s)\n'.format(k, median(data[k]), stdev(data[k]))) |