Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | from __future__ import print_function |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 4 | |
| 5 | import time, subprocess, sys, os, shutil, glob, random |
| 6 | import argparse |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 7 | |
| 8 | def median(lst): |
| 9 | lst = sorted(lst) |
| 10 | mid, odd = divmod(len(lst), 2) |
| 11 | if odd: |
| 12 | return lst[mid] |
| 13 | else: |
| 14 | return (lst[mid - 1] + lst[mid]) / 2.0 |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 15 | |
Martin Hořeňovský | 2049113 | 2017-02-14 15:34:00 +0100 | [diff] [blame] | 16 | def mean(lst): |
| 17 | return float(sum(lst)) / max(len(lst), 1) |
| 18 | |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 19 | compiler_path = '' |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 20 | flags = [] |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 21 | |
| 22 | main_file = r''' |
| 23 | #define CATCH_CONFIG_MAIN |
| 24 | #include "catch.hpp" |
| 25 | ''' |
| 26 | main_name = 'catch-main.cpp' |
| 27 | |
| 28 | dir_name = 'benchmark-dir' |
| 29 | |
| 30 | files = 20 |
| 31 | test_cases_in_file = 20 |
| 32 | sections_in_file = 4 |
| 33 | assertions_per_section = 5 |
| 34 | |
| 35 | checks = [ |
| 36 | 'a != b', 'a != c', 'a != d', 'a != e', 'b != c', 'b != d', 'b != e', 'c != d', 'c != e', 'd != e', 'a + a == a', |
| 37 | 'a + b == b', 'a + c == c', 'a + d == d', 'a + e == e', 'b + a == b', 'b + b == c', 'b + c == d', |
| 38 | 'b + d == e', 'c + a == c', 'c + b == d', 'c + c == e', 'd + a == d', 'd + b == e', 'e + a == e', |
| 39 | 'a + a + a == a', 'b + c == a + d', 'c + a + a == a + b + b + a', |
| 40 | 'a < b', 'b < c', 'c < d', 'd < e', 'a >= a', 'd >= b', |
| 41 | ] |
| 42 | |
| 43 | def create_temp_dir(): |
| 44 | if os.path.exists(dir_name): |
| 45 | shutil.rmtree(dir_name) |
| 46 | os.mkdir(dir_name) |
| 47 | |
| 48 | def copy_catch(path_to_catch): |
| 49 | shutil.copy(path_to_catch, dir_name) |
| 50 | |
| 51 | def create_catch_main(): |
| 52 | with open(main_name, 'w') as f: |
| 53 | f.write(main_file) |
| 54 | |
| 55 | def compile_main(): |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 56 | start_t = time.time() |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 57 | subprocess.check_call([compiler_path, main_name, '-c'] + flags) |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 58 | end_t = time.time() |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 59 | return end_t - start_t |
| 60 | |
| 61 | def compile_files(): |
Martin Hořeňovský | 97edf7c | 2017-08-30 18:11:52 +0200 | [diff] [blame] | 62 | cpp_files = glob.glob('tests*.cpp') |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 63 | start_t = time.time() |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 64 | subprocess.check_call([compiler_path, '-c'] + flags + cpp_files) |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 65 | end_t = time.time() |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 66 | return end_t - start_t |
| 67 | |
| 68 | def link_files(): |
| 69 | obj_files = glob.glob('*.o') |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 70 | start_t = time.time() |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 71 | subprocess.check_call([compiler_path] + flags + obj_files) |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 72 | end_t = time.time() |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 73 | return end_t - start_t |
| 74 | |
| 75 | def benchmark(func): |
Martin Hořeňovský | 2049113 | 2017-02-14 15:34:00 +0100 | [diff] [blame] | 76 | results = [func() for i in range(10)] |
| 77 | return mean(results), median(results) |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 78 | |
| 79 | def char_range(start, end): |
| 80 | for c in range(ord(start), ord(end)): |
| 81 | yield chr(c) |
| 82 | |
| 83 | def generate_sections(fd): |
| 84 | for i in range(sections_in_file): |
| 85 | fd.write(' SECTION("Section {}") {{\n'.format(i)) |
| 86 | fd.write('\n'.join(' CHECK({});'.format(check) for check in random.sample(checks, assertions_per_section))) |
| 87 | fd.write(' }\n') |
| 88 | |
| 89 | |
| 90 | def generate_file(file_no): |
| 91 | with open('tests{}.cpp'.format(file_no), 'w') as f: |
| 92 | f.write('#include "catch.hpp"\n\n') |
| 93 | for i in range(test_cases_in_file): |
| 94 | f.write('TEST_CASE("File {} test {}", "[.compile]"){{\n'.format(file_no, i)) |
| 95 | for i, c in enumerate(char_range('a', 'f')): |
| 96 | f.write(' int {} = {};\n'.format(c, i)) |
| 97 | generate_sections(f) |
| 98 | f.write('}\n\n') |
| 99 | |
| 100 | |
| 101 | def generate_files(): |
| 102 | create_catch_main() |
| 103 | for i in range(files): |
| 104 | generate_file(i) |
| 105 | |
| 106 | |
| 107 | options = ['all', 'main', 'files', 'link'] |
| 108 | |
| 109 | parser = argparse.ArgumentParser(description='Benchmarks Catch\'s compile times against some synthetic tests') |
| 110 | # Add first arg -- benchmark type |
| 111 | parser.add_argument('benchmark_kind', nargs='?', default='all', choices=options, help='What kind of benchmark to run, default: all') |
| 112 | |
| 113 | # Args to allow changing header/compiler |
| 114 | parser.add_argument('-I', '--catch-header', default='catch.hpp', help = 'Path to catch.hpp, default: catch.hpp') |
| 115 | parser.add_argument('-c', '--compiler', default='g++', help = 'Compiler to use, default: g++') |
| 116 | |
Martin Hořeňovský | 97edf7c | 2017-08-30 18:11:52 +0200 | [diff] [blame] | 117 | parser.add_argument('-f', '--flags', help = 'Flags to be passed to the compiler. Pass as "," separated list') |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 118 | |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 119 | # Allow creating files only, without running the whole thing |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 120 | parser.add_argument('-g', '--generate-files', action='store_true', help='Generate test files and quit') |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 121 | |
| 122 | args = parser.parse_args() |
| 123 | |
| 124 | compiler_path = args.compiler |
| 125 | catch_path = args.catch_header |
| 126 | |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 127 | if args.generate_files: |
| 128 | create_temp_dir() |
| 129 | copy_catch(catch_path) |
Martin Hořeňovský | 374c050 | 2017-03-06 10:59:17 +0100 | [diff] [blame] | 130 | os.chdir(dir_name) |
Martin Hořeňovský | 0837132 | 2017-02-12 12:25:43 +0100 | [diff] [blame] | 131 | # now create the fake test files |
| 132 | generate_files() |
| 133 | # Early exit |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 134 | print('Finished generating files') |
| 135 | exit(1) |
| 136 | |
Martin Hořeňovský | fd6c7ae | 2017-03-02 18:27:31 +0100 | [diff] [blame] | 137 | os.chdir(dir_name) |
| 138 | |
Martin Hořeňovský | 6da5e08 | 2017-02-14 15:34:17 +0100 | [diff] [blame] | 139 | if args.flags: |
Martin Hořeňovský | 97edf7c | 2017-08-30 18:11:52 +0200 | [diff] [blame] | 140 | flags = args.flags.split(',') |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 141 | |
| 142 | print('Time needed for ...') |
| 143 | if args.benchmark_kind in ('all', 'main'): |
Martin Hořeňovský | 2049113 | 2017-02-14 15:34:00 +0100 | [diff] [blame] | 144 | print(' ... compiling main, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_main))) |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 145 | if args.benchmark_kind in ('all', 'files'): |
Martin Hořeňovský | 2049113 | 2017-02-14 15:34:00 +0100 | [diff] [blame] | 146 | print(' ... compiling test files, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_files))) |
Martin Hořeňovský | 26df078 | 2017-02-08 14:14:01 +0100 | [diff] [blame] | 147 | if args.benchmark_kind in ('all', 'link'): |
Martin Hořeňovský | 2049113 | 2017-02-14 15:34:00 +0100 | [diff] [blame] | 148 | print(' ... linking everything, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(link_files))) |