Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 1 | """Benchmark some basic import use-cases. |
| 2 | |
| 3 | The assumption is made that this benchmark is run in a fresh interpreter and |
| 4 | thus has no external changes made to import-related attributes in sys. |
| 5 | |
| 6 | """ |
Brett Cannon | d382bfc | 2012-07-20 14:54:53 -0400 | [diff] [blame] | 7 | from test.test_importlib import util |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 8 | import decimal |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 9 | import imp |
| 10 | import importlib |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 11 | import importlib.machinery |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 12 | import json |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 13 | import os |
| 14 | import py_compile |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 15 | import sys |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 16 | import tabnanny |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 17 | import timeit |
| 18 | |
| 19 | |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 20 | def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3): |
| 21 | """Bench the given statement as many times as necessary until total |
| 22 | executions take one second.""" |
| 23 | stmt = "__import__({!r})".format(name) |
| 24 | timer = timeit.Timer(stmt) |
| 25 | for x in range(repeat): |
| 26 | total_time = 0 |
| 27 | count = 0 |
| 28 | while total_time < seconds: |
| 29 | try: |
| 30 | total_time += timer.timeit(1) |
| 31 | finally: |
| 32 | cleanup() |
| 33 | count += 1 |
| 34 | else: |
| 35 | # One execution too far |
| 36 | if total_time > seconds: |
| 37 | count -= 1 |
Brett Cannon | 7b9bcb8 | 2010-07-15 06:24:04 +0000 | [diff] [blame] | 38 | yield count // seconds |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 39 | |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 40 | def from_cache(seconds, repeat): |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 41 | """sys.modules""" |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 42 | name = '<benchmark import>' |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 43 | module = imp.new_module(name) |
| 44 | module.__file__ = '<test>' |
| 45 | module.__package__ = '' |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 46 | with util.uncache(name): |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 47 | sys.modules[name] = module |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 48 | yield from bench(name, repeat=repeat, seconds=seconds) |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 49 | |
| 50 | |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 51 | def builtin_mod(seconds, repeat): |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 52 | """Built-in module""" |
| 53 | name = 'errno' |
| 54 | if name in sys.modules: |
| 55 | del sys.modules[name] |
Brett Cannon | 7b9bcb8 | 2010-07-15 06:24:04 +0000 | [diff] [blame] | 56 | # Relying on built-in importer being implicit. |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 57 | yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, |
| 58 | seconds=seconds) |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 59 | |
| 60 | |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 61 | def source_wo_bytecode(seconds, repeat): |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 62 | """Source w/o bytecode: small""" |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 63 | sys.dont_write_bytecode = True |
| 64 | try: |
| 65 | name = '__importlib_test_benchmark__' |
| 66 | # Clears out sys.modules and puts an entry at the front of sys.path. |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 67 | with util.create_modules(name) as mapping: |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 68 | assert not os.path.exists(imp.cache_from_source(mapping[name])) |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 69 | sys.meta_path.append(importlib.machinery.PathFinder) |
| 70 | loader = (importlib.machinery.SourceFileLoader, |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 71 | importlib.machinery.SOURCE_SUFFIXES) |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 72 | sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 73 | yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, |
| 74 | seconds=seconds) |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 75 | finally: |
| 76 | sys.dont_write_bytecode = False |
| 77 | |
| 78 | |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 79 | def _wo_bytecode(module): |
| 80 | name = module.__name__ |
| 81 | def benchmark_wo_bytecode(seconds, repeat): |
| 82 | """Source w/o bytecode: {}""" |
| 83 | bytecode_path = imp.cache_from_source(module.__file__) |
| 84 | if os.path.exists(bytecode_path): |
| 85 | os.unlink(bytecode_path) |
| 86 | sys.dont_write_bytecode = True |
| 87 | try: |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 88 | yield from bench(name, lambda: sys.modules.pop(name), |
| 89 | repeat=repeat, seconds=seconds) |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 90 | finally: |
| 91 | sys.dont_write_bytecode = False |
| 92 | |
| 93 | benchmark_wo_bytecode.__doc__ = benchmark_wo_bytecode.__doc__.format(name) |
| 94 | return benchmark_wo_bytecode |
| 95 | |
| 96 | tabnanny_wo_bytecode = _wo_bytecode(tabnanny) |
| 97 | decimal_wo_bytecode = _wo_bytecode(decimal) |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 98 | |
| 99 | |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 100 | def source_writing_bytecode(seconds, repeat): |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 101 | """Source writing bytecode: small""" |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 102 | assert not sys.dont_write_bytecode |
| 103 | name = '__importlib_test_benchmark__' |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 104 | with util.create_modules(name) as mapping: |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 105 | sys.meta_path.append(importlib.machinery.PathFinder) |
| 106 | loader = (importlib.machinery.SourceFileLoader, |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 107 | importlib.machinery.SOURCE_SUFFIXES) |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 108 | sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 109 | def cleanup(): |
| 110 | sys.modules.pop(name) |
| 111 | os.unlink(imp.cache_from_source(mapping[name])) |
| 112 | for result in bench(name, cleanup, repeat=repeat, seconds=seconds): |
| 113 | assert not os.path.exists(imp.cache_from_source(mapping[name])) |
| 114 | yield result |
| 115 | |
| 116 | |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 117 | def _writing_bytecode(module): |
| 118 | name = module.__name__ |
| 119 | def writing_bytecode_benchmark(seconds, repeat): |
| 120 | """Source writing bytecode: {}""" |
| 121 | assert not sys.dont_write_bytecode |
| 122 | def cleanup(): |
| 123 | sys.modules.pop(name) |
| 124 | os.unlink(imp.cache_from_source(module.__file__)) |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 125 | yield from bench(name, cleanup, repeat=repeat, seconds=seconds) |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 126 | |
| 127 | writing_bytecode_benchmark.__doc__ = ( |
| 128 | writing_bytecode_benchmark.__doc__.format(name)) |
| 129 | return writing_bytecode_benchmark |
| 130 | |
| 131 | tabnanny_writing_bytecode = _writing_bytecode(tabnanny) |
| 132 | decimal_writing_bytecode = _writing_bytecode(decimal) |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 133 | |
| 134 | |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 135 | def source_using_bytecode(seconds, repeat): |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 136 | """Source w/ bytecode: small""" |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 137 | name = '__importlib_test_benchmark__' |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 138 | with util.create_modules(name) as mapping: |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 139 | sys.meta_path.append(importlib.machinery.PathFinder) |
| 140 | loader = (importlib.machinery.SourceFileLoader, |
Serhiy Storchaka | eb51faa | 2016-06-30 10:33:17 +0300 | [diff] [blame] | 141 | importlib.machinery.SOURCE_SUFFIXES) |
Brett Cannon | 810c64d | 2012-05-11 11:12:00 -0400 | [diff] [blame] | 142 | sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 143 | py_compile.compile(mapping[name]) |
| 144 | assert os.path.exists(imp.cache_from_source(mapping[name])) |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 145 | yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, |
| 146 | seconds=seconds) |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 147 | |
| 148 | |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 149 | def _using_bytecode(module): |
| 150 | name = module.__name__ |
| 151 | def using_bytecode_benchmark(seconds, repeat): |
| 152 | """Source w/ bytecode: {}""" |
| 153 | py_compile.compile(module.__file__) |
Philip Jenvey | fd0d3e5 | 2012-10-01 15:34:31 -0700 | [diff] [blame] | 154 | yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, |
| 155 | seconds=seconds) |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 156 | |
| 157 | using_bytecode_benchmark.__doc__ = ( |
| 158 | using_bytecode_benchmark.__doc__.format(name)) |
| 159 | return using_bytecode_benchmark |
| 160 | |
| 161 | tabnanny_using_bytecode = _using_bytecode(tabnanny) |
| 162 | decimal_using_bytecode = _using_bytecode(decimal) |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 163 | |
| 164 | |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 165 | def main(import_, options): |
| 166 | if options.source_file: |
| 167 | with options.source_file: |
| 168 | prev_results = json.load(options.source_file) |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 169 | else: |
| 170 | prev_results = {} |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 171 | __builtins__.__import__ = import_ |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 172 | benchmarks = (from_cache, builtin_mod, |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 173 | source_writing_bytecode, |
Brett Cannon | cae1068 | 2012-02-07 09:40:33 -0500 | [diff] [blame] | 174 | source_wo_bytecode, source_using_bytecode, |
Brett Cannon | 466e6a9 | 2012-02-07 09:19:12 -0500 | [diff] [blame] | 175 | tabnanny_writing_bytecode, |
Brett Cannon | cae1068 | 2012-02-07 09:40:33 -0500 | [diff] [blame] | 176 | tabnanny_wo_bytecode, tabnanny_using_bytecode, |
| 177 | decimal_writing_bytecode, |
| 178 | decimal_wo_bytecode, decimal_using_bytecode, |
| 179 | ) |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 180 | if options.benchmark: |
Brett Cannon | e3a9ae5 | 2012-01-30 19:27:51 -0500 | [diff] [blame] | 181 | for b in benchmarks: |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 182 | if b.__doc__ == options.benchmark: |
Brett Cannon | e3a9ae5 | 2012-01-30 19:27:51 -0500 | [diff] [blame] | 183 | benchmarks = [b] |
| 184 | break |
| 185 | else: |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 186 | print('Unknown benchmark: {!r}'.format(options.benchmark, |
| 187 | file=sys.stderr)) |
Brett Cannon | e3a9ae5 | 2012-01-30 19:27:51 -0500 | [diff] [blame] | 188 | sys.exit(1) |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 189 | seconds = 1 |
| 190 | seconds_plural = 's' if seconds > 1 else '' |
| 191 | repeat = 3 |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 192 | header = ('Measuring imports/second over {} second{}, best out of {}\n' |
Brett Cannon | cae1068 | 2012-02-07 09:40:33 -0500 | [diff] [blame] | 193 | 'Entire benchmark run should take about {} seconds\n' |
| 194 | 'Using {!r} as __import__\n') |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 195 | print(header.format(seconds, seconds_plural, repeat, |
Brett Cannon | cae1068 | 2012-02-07 09:40:33 -0500 | [diff] [blame] | 196 | len(benchmarks) * seconds * repeat, __import__)) |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 197 | new_results = {} |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 198 | for benchmark in benchmarks: |
| 199 | print(benchmark.__doc__, "[", end=' ') |
| 200 | sys.stdout.flush() |
| 201 | results = [] |
Brett Cannon | 5db0c94 | 2010-07-22 07:40:56 +0000 | [diff] [blame] | 202 | for result in benchmark(seconds=seconds, repeat=repeat): |
Brett Cannon | 23cf574 | 2009-09-03 20:45:21 +0000 | [diff] [blame] | 203 | results.append(result) |
| 204 | print(result, end=' ') |
| 205 | sys.stdout.flush() |
Brett Cannon | 3b0a19e | 2010-07-16 19:04:29 +0000 | [diff] [blame] | 206 | assert not sys.dont_write_bytecode |
Brett Cannon | cbe1a4e | 2010-07-16 19:26:23 +0000 | [diff] [blame] | 207 | print("]", "best is", format(max(results), ',d')) |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 208 | new_results[benchmark.__doc__] = results |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 209 | if prev_results: |
| 210 | print('\n\nComparing new vs. old\n') |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 211 | for benchmark in benchmarks: |
| 212 | benchmark_name = benchmark.__doc__ |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 213 | old_result = max(prev_results[benchmark_name]) |
| 214 | new_result = max(new_results[benchmark_name]) |
| 215 | result = '{:,d} vs. {:,d} ({:%})'.format(new_result, |
| 216 | old_result, |
| 217 | new_result/old_result) |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 218 | print(benchmark_name, ':', result) |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 219 | if options.dest_file: |
| 220 | with options.dest_file: |
| 221 | json.dump(new_results, options.dest_file, indent=2) |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 222 | |
| 223 | |
| 224 | if __name__ == '__main__': |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 225 | import argparse |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 226 | |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 227 | parser = argparse.ArgumentParser() |
| 228 | parser.add_argument('-b', '--builtin', dest='builtin', action='store_true', |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 229 | default=False, help="use the built-in __import__") |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 230 | parser.add_argument('-r', '--read', dest='source_file', |
| 231 | type=argparse.FileType('r'), |
| 232 | help='file to read benchmark data from to compare ' |
| 233 | 'against') |
| 234 | parser.add_argument('-w', '--write', dest='dest_file', |
| 235 | type=argparse.FileType('w'), |
| 236 | help='file to write benchmark data to') |
Brett Cannon | e3a9ae5 | 2012-01-30 19:27:51 -0500 | [diff] [blame] | 237 | parser.add_argument('--benchmark', dest='benchmark', |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 238 | help='specific benchmark to run') |
Brett Cannon | 190f33c | 2012-01-30 19:12:29 -0500 | [diff] [blame] | 239 | options = parser.parse_args() |
Brett Cannon | 6ae7a7d | 2009-03-30 15:53:01 +0000 | [diff] [blame] | 240 | import_ = __import__ |
| 241 | if not options.builtin: |
| 242 | import_ = importlib.__import__ |
| 243 | |
Brett Cannon | dfc3270 | 2012-02-23 19:34:35 -0500 | [diff] [blame] | 244 | main(import_, options) |