Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 1 | # Copyright 2016, Google Inc. |
| 2 | # All rights reserved. |
| 3 | # |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following disclaimer |
| 12 | # in the documentation and/or other materials provided with the |
| 13 | # distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived from |
| 16 | # this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | # performance scenario configuration for various languages |
| 31 | |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 32 | import math |
| 33 | |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 34 | WARMUP_SECONDS=5 |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 35 | JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 36 | BENCHMARK_SECONDS=30 |
| 37 | |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 38 | SMOKETEST='smoketest' |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 39 | SCALABLE='scalable' |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 40 | SWEEP='sweep' |
Craig Tiller | b6df247 | 2016-09-13 09:41:26 -0700 | [diff] [blame] | 41 | DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST] |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 42 | |
Jan Tattermusch | 299f97f | 2016-04-20 18:41:55 -0700 | [diff] [blame] | 43 | SECURE_SECARGS = {'use_test_ca': True, |
| 44 | 'server_host_override': 'foo.test.google.fr'} |
| 45 | |
Jan Tattermusch | 4fec48b | 2016-04-12 15:12:54 -0700 | [diff] [blame] | 46 | HISTOGRAM_PARAMS = { |
| 47 | 'resolution': 0.01, |
| 48 | 'max_possible': 60e9, |
| 49 | } |
| 50 | |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 51 | # target number of RPCs outstanding on across all client channels in |
| 52 | # non-ping-pong tests (since we can only specify per-channel numbers, the |
| 53 | # actual target will be slightly higher) |
| 54 | OUTSTANDING_REQUESTS={ |
Craig Tiller | 6c38c1b | 2016-09-13 08:34:27 -0700 | [diff] [blame] | 55 | 'async': 6400, |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 56 | 'sync': 1000 |
| 57 | } |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 58 | |
| 59 | # wide is the number of client channels in multi-channel tests (1 otherwise) |
| 60 | WIDE=64 |
| 61 | |
| 62 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 63 | def _get_secargs(is_secure): |
| 64 | if is_secure: |
| 65 | return SECURE_SECARGS |
| 66 | else: |
| 67 | return None |
| 68 | |
| 69 | |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 70 | def remove_nonproto_fields(scenario): |
| 71 | """Remove special-purpose that contains some extra info about the scenario |
| 72 | but don't belong to the ScenarioConfig protobuf message""" |
| 73 | scenario.pop('CATEGORIES', None) |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 74 | scenario.pop('CLIENT_LANGUAGE', None) |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 75 | scenario.pop('SERVER_LANGUAGE', None) |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 76 | scenario.pop('EXCLUDED_POLL_ENGINES', None) |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 77 | return scenario |
| 78 | |
| 79 | |
Craig Tiller | c5aa700 | 2016-09-13 09:36:09 -0700 | [diff] [blame] | 80 | def geometric_progression(start, stop, step): |
| 81 | n = start |
| 82 | while n < stop: |
| 83 | yield int(round(n)) |
| 84 | n *= step |
| 85 | |
| 86 | |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 87 | def _payload_type(use_generic_payload, req_size, resp_size): |
| 88 | r = {} |
| 89 | sizes = { |
| 90 | 'req_size': req_size, |
| 91 | 'resp_size': resp_size, |
| 92 | } |
| 93 | if use_generic_payload: |
| 94 | r['bytebuf_params'] = sizes |
| 95 | else: |
| 96 | r['simple_params'] = sizes |
Alexander Polcyn | c6e333e | 2017-02-03 09:26:35 -0800 | [diff] [blame] | 97 | return r |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 98 | |
| 99 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 100 | def _ping_pong_scenario(name, rpc_type, |
| 101 | client_type, server_type, |
| 102 | secure=True, |
| 103 | use_generic_payload=False, |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 104 | req_size=0, |
| 105 | resp_size=0, |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 106 | unconstrained_client=None, |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 107 | client_language=None, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 108 | server_language=None, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 109 | async_server_threads=0, |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 110 | warmup_seconds=WARMUP_SECONDS, |
Craig Tiller | b6df247 | 2016-09-13 09:41:26 -0700 | [diff] [blame] | 111 | categories=DEFAULT_CATEGORIES, |
Craig Tiller | c5aa700 | 2016-09-13 09:36:09 -0700 | [diff] [blame] | 112 | channels=None, |
Craig Tiller | 0c80c7d | 2016-09-28 16:30:55 -0700 | [diff] [blame] | 113 | outstanding=None, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 114 | resource_quota_size=None, |
| 115 | excluded_poll_engines=[]): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 116 | """Creates a basic ping pong scenario.""" |
| 117 | scenario = { |
| 118 | 'name': name, |
| 119 | 'num_servers': 1, |
| 120 | 'num_clients': 1, |
| 121 | 'client_config': { |
| 122 | 'client_type': client_type, |
| 123 | 'security_params': _get_secargs(secure), |
| 124 | 'outstanding_rpcs_per_channel': 1, |
| 125 | 'client_channels': 1, |
| 126 | 'async_client_threads': 1, |
| 127 | 'rpc_type': rpc_type, |
| 128 | 'load_params': { |
| 129 | 'closed_loop': {} |
| 130 | }, |
| 131 | 'histogram_params': HISTOGRAM_PARAMS, |
| 132 | }, |
| 133 | 'server_config': { |
| 134 | 'server_type': server_type, |
| 135 | 'security_params': _get_secargs(secure), |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 136 | 'async_server_threads': async_server_threads, |
| 137 | }, |
| 138 | 'warmup_seconds': warmup_seconds, |
| 139 | 'benchmark_seconds': BENCHMARK_SECONDS |
| 140 | } |
Craig Tiller | 20afa3d | 2016-10-17 14:52:14 -0700 | [diff] [blame] | 141 | if resource_quota_size: |
| 142 | scenario['server_config']['resource_quota_size'] = resource_quota_size |
Jan Tattermusch | 28f6c01 | 2016-05-22 21:23:43 -0400 | [diff] [blame] | 143 | if use_generic_payload: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 144 | if server_type != 'ASYNC_GENERIC_SERVER': |
| 145 | raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') |
Alexander Polcyn | f797c65 | 2017-02-08 14:29:30 -0800 | [diff] [blame] | 146 | scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) |
| 147 | |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 148 | scenario['client_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 149 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 150 | if unconstrained_client: |
Craig Tiller | 0c98675 | 2016-09-13 10:01:26 -0700 | [diff] [blame] | 151 | outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] |
Craig Tiller | faa835c | 2017-01-18 15:28:44 -0800 | [diff] [blame] | 152 | # clamp buffer usage to something reasonable (16 gig for now) |
| 153 | MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 |
| 154 | if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: |
| 155 | outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, resp_size)) |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 156 | wide = channels if channels is not None else WIDE |
Craig Tiller | 0c98675 | 2016-09-13 10:01:26 -0700 | [diff] [blame] | 157 | deep = int(math.ceil(1.0 * outstanding_calls / wide)) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 158 | |
Jan Tattermusch | 28f6c01 | 2016-05-22 21:23:43 -0400 | [diff] [blame] | 159 | scenario['num_clients'] = 0 # use as many client as available. |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 160 | scenario['client_config']['outstanding_rpcs_per_channel'] = deep |
| 161 | scenario['client_config']['client_channels'] = wide |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 162 | scenario['client_config']['async_client_threads'] = 0 |
| 163 | else: |
| 164 | scenario['client_config']['outstanding_rpcs_per_channel'] = 1 |
| 165 | scenario['client_config']['client_channels'] = 1 |
| 166 | scenario['client_config']['async_client_threads'] = 1 |
| 167 | |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 168 | if client_language: |
| 169 | # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py |
| 170 | scenario['CLIENT_LANGUAGE'] = client_language |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 171 | if server_language: |
| 172 | # the SERVER_LANGUAGE field is recognized by run_performance_tests.py |
| 173 | scenario['SERVER_LANGUAGE'] = server_language |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 174 | if categories: |
| 175 | scenario['CATEGORIES'] = categories |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 176 | if len(excluded_poll_engines): |
| 177 | # The polling engines for which this scenario is excluded |
| 178 | scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 179 | return scenario |
| 180 | |
| 181 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 182 | class CXXLanguage: |
| 183 | |
| 184 | def __init__(self): |
| 185 | self.safename = 'cxx' |
| 186 | |
| 187 | def worker_cmdline(self): |
| 188 | return ['bins/opt/qps_worker'] |
| 189 | |
| 190 | def worker_port_offset(self): |
| 191 | return 0 |
| 192 | |
| 193 | def scenarios(self): |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 194 | # TODO(ctiller): add 70% load latency test |
| 195 | for secure in [True, False]: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 196 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 197 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 198 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 199 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 200 | 'cpp_generic_async_streaming_ping_pong_%s' % secstr, |
| 201 | rpc_type='STREAMING', |
| 202 | client_type='ASYNC_CLIENT', |
| 203 | server_type='ASYNC_GENERIC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 204 | use_generic_payload=True, async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 205 | secure=secure, |
| 206 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 207 | |
| 208 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 209 | 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, |
| 210 | rpc_type='STREAMING', |
| 211 | client_type='ASYNC_CLIENT', |
| 212 | server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 213 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 214 | secure=secure, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 215 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 216 | |
| 217 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 218 | 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, |
| 219 | rpc_type='STREAMING', |
| 220 | client_type='ASYNC_CLIENT', |
| 221 | server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 222 | unconstrained_client='async', use_generic_payload=True, |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 223 | async_server_threads=1, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 224 | secure=secure) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 225 | |
Sree Kuchibhotla | 923d641 | 2016-09-26 19:32:17 -0700 | [diff] [blame] | 226 | yield _ping_pong_scenario( |
| 227 | 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' % |
| 228 | (secstr), |
| 229 | rpc_type='UNARY', |
| 230 | client_type='ASYNC_CLIENT', |
| 231 | server_type='SYNC_SERVER', |
| 232 | unconstrained_client='async', |
| 233 | secure=secure, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 234 | categories=smoketest_categories + [SCALABLE], |
| 235 | excluded_poll_engines = ['poll-cv']) |
Sree Kuchibhotla | 923d641 | 2016-09-26 19:32:17 -0700 | [diff] [blame] | 236 | |
Sree Kuchibhotla | a83ee60 | 2016-10-25 11:54:52 -0700 | [diff] [blame] | 237 | yield _ping_pong_scenario( |
| 238 | 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' % secstr, |
| 239 | rpc_type='STREAMING', |
| 240 | client_type='ASYNC_CLIENT', |
| 241 | server_type='SYNC_SERVER', |
| 242 | unconstrained_client='async', |
| 243 | secure=secure, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 244 | categories=smoketest_categories+[SCALABLE], |
| 245 | excluded_poll_engines = ['poll-cv']) |
Sree Kuchibhotla | a83ee60 | 2016-10-25 11:54:52 -0700 | [diff] [blame] | 246 | |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 247 | for rpc_type in ['unary', 'streaming']: |
| 248 | for synchronicity in ['sync', 'async']: |
| 249 | yield _ping_pong_scenario( |
| 250 | 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr), |
| 251 | rpc_type=rpc_type.upper(), |
| 252 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 253 | server_type='%s_SERVER' % synchronicity.upper(), |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 254 | async_server_threads=1, |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 255 | secure=secure) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 256 | |
Craig Tiller | faa835c | 2017-01-18 15:28:44 -0800 | [diff] [blame] | 257 | for size in geometric_progression(1, 1024*1024*1024+1, 8): |
| 258 | yield _ping_pong_scenario( |
| 259 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % (synchronicity, rpc_type, secstr, size), |
| 260 | rpc_type=rpc_type.upper(), |
| 261 | req_size=size, |
| 262 | resp_size=size, |
| 263 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 264 | server_type='%s_SERVER' % synchronicity.upper(), |
| 265 | unconstrained_client=synchronicity, |
| 266 | secure=secure, |
| 267 | categories=[SWEEP]) |
| 268 | |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 269 | yield _ping_pong_scenario( |
| 270 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr), |
| 271 | rpc_type=rpc_type.upper(), |
| 272 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 273 | server_type='%s_SERVER' % synchronicity.upper(), |
| 274 | unconstrained_client=synchronicity, |
| 275 | secure=secure, |
| 276 | categories=smoketest_categories+[SCALABLE]) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 277 | |
Vijay Pai | a582def | 2017-03-09 05:47:46 -0800 | [diff] [blame] | 278 | # TODO(vjpai): Re-enable this test. It has a lot of timeouts |
| 279 | # and hasn't yet been conclusively identified as a test failure |
| 280 | # or race in the library |
| 281 | # yield _ping_pong_scenario( |
| 282 | # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), |
| 283 | # rpc_type=rpc_type.upper(), |
| 284 | # client_type='%s_CLIENT' % synchronicity.upper(), |
| 285 | # server_type='%s_SERVER' % synchronicity.upper(), |
| 286 | # unconstrained_client=synchronicity, |
| 287 | # secure=secure, |
| 288 | # categories=smoketest_categories+[SCALABLE], |
| 289 | # resource_quota_size=500*1024) |
Craig Tiller | 0c80c7d | 2016-09-28 16:30:55 -0700 | [diff] [blame] | 290 | |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 291 | for channels in geometric_progression(1, 20000, math.sqrt(10)): |
| 292 | for outstanding in geometric_progression(1, 200000, math.sqrt(10)): |
| 293 | if synchronicity == 'sync' and outstanding > 1200: continue |
| 294 | if outstanding < channels: continue |
| 295 | yield _ping_pong_scenario( |
| 296 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding), |
| 297 | rpc_type=rpc_type.upper(), |
| 298 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 299 | server_type='%s_SERVER' % synchronicity.upper(), |
| 300 | unconstrained_client=synchronicity, secure=secure, |
| 301 | categories=[SWEEP], channels=channels, outstanding=outstanding) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 302 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 303 | def __str__(self): |
| 304 | return 'c++' |
| 305 | |
| 306 | |
| 307 | class CSharpLanguage: |
| 308 | |
| 309 | def __init__(self): |
| 310 | self.safename = str(self) |
| 311 | |
| 312 | def worker_cmdline(self): |
| 313 | return ['tools/run_tests/performance/run_worker_csharp.sh'] |
| 314 | |
| 315 | def worker_port_offset(self): |
| 316 | return 100 |
| 317 | |
| 318 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 319 | yield _ping_pong_scenario( |
| 320 | 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', |
| 321 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 322 | use_generic_payload=True, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 323 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 324 | |
| 325 | yield _ping_pong_scenario( |
| 326 | 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 327 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
| 328 | |
| 329 | yield _ping_pong_scenario( |
| 330 | 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 331 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 332 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 333 | |
| 334 | yield _ping_pong_scenario( |
| 335 | 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', |
| 336 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') |
| 337 | |
Jan Tattermusch | 16713b9 | 2016-05-09 14:34:11 -0700 | [diff] [blame] | 338 | yield _ping_pong_scenario( |
| 339 | 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 340 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 341 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 342 | categories=[SMOKETEST,SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 343 | |
Jan Tattermusch | 16713b9 | 2016-05-09 14:34:11 -0700 | [diff] [blame] | 344 | yield _ping_pong_scenario( |
| 345 | 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', |
| 346 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 347 | unconstrained_client='async', |
| 348 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 349 | |
Jan Tattermusch | abd1b38 | 2016-05-08 18:47:40 -0700 | [diff] [blame] | 350 | yield _ping_pong_scenario( |
| 351 | 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
| 352 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 353 | server_language='c++', async_server_threads=1, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 354 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 355 | |
| 356 | yield _ping_pong_scenario( |
| 357 | 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 358 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 359 | server_language='c++', async_server_threads=1) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 360 | |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 361 | yield _ping_pong_scenario( |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 362 | 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 363 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 364 | unconstrained_client='async', server_language='c++', |
| 365 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 366 | |
| 367 | yield _ping_pong_scenario( |
| 368 | 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 369 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 370 | unconstrained_client='sync', server_language='c++', |
| 371 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 372 | |
| 373 | yield _ping_pong_scenario( |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 374 | 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 375 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 376 | unconstrained_client='async', client_language='c++', |
| 377 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 378 | |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 379 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 380 | def __str__(self): |
| 381 | return 'csharp' |
| 382 | |
| 383 | |
| 384 | class NodeLanguage: |
| 385 | |
| 386 | def __init__(self): |
| 387 | pass |
| 388 | self.safename = str(self) |
| 389 | |
| 390 | def worker_cmdline(self): |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 391 | return ['tools/run_tests/performance/run_worker_node.sh', |
| 392 | '--benchmark_impl=grpc'] |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 393 | |
| 394 | def worker_port_offset(self): |
| 395 | return 200 |
| 396 | |
| 397 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 398 | # TODO(jtattermusch): make this scenario work |
| 399 | #yield _ping_pong_scenario( |
| 400 | # 'node_generic_async_streaming_ping_pong', rpc_type='STREAMING', |
| 401 | # client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 402 | # use_generic_payload=True) |
| 403 | |
| 404 | # TODO(jtattermusch): make this scenario work |
| 405 | #yield _ping_pong_scenario( |
| 406 | # 'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 407 | # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
| 408 | |
| 409 | yield _ping_pong_scenario( |
| 410 | 'node_protobuf_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 411 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 412 | categories=[SCALABLE, SMOKETEST]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 413 | |
| 414 | yield _ping_pong_scenario( |
murgatroid99 | 9030c81 | 2016-09-16 13:25:08 -0700 | [diff] [blame] | 415 | 'cpp_to_node_unary_ping_pong', rpc_type='UNARY', |
| 416 | client_type='ASYNC_CLIENT', server_type='async_server', |
| 417 | client_language='c++') |
| 418 | |
murgatroid99 | faaa8b8 | 2017-03-02 10:50:34 -0800 | [diff] [blame] | 419 | # TODO(murgatroid99): fix bugs with this scenario and re-enable it |
| 420 | # yield _ping_pong_scenario( |
| 421 | # 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 422 | # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 423 | # unconstrained_client='async', |
| 424 | # categories=[SCALABLE, SMOKETEST]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 425 | |
| 426 | # TODO(jtattermusch): make this scenario work |
| 427 | #yield _ping_pong_scenario( |
| 428 | # 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', |
| 429 | # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 430 | # unconstrained_client='async') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 431 | |
| 432 | # TODO(jtattermusch): make this scenario work |
| 433 | #yield _ping_pong_scenario( |
| 434 | # 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
| 435 | # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 436 | # server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 437 | |
| 438 | # TODO(jtattermusch): make this scenario work |
| 439 | #yield _ping_pong_scenario( |
| 440 | # 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 441 | # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 442 | # server_language='c++', async_server_threads=1) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 443 | |
| 444 | def __str__(self): |
| 445 | return 'node' |
| 446 | |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 447 | class PythonLanguage: |
| 448 | |
| 449 | def __init__(self): |
| 450 | self.safename = 'python' |
| 451 | |
| 452 | def worker_cmdline(self): |
| 453 | return ['tools/run_tests/performance/run_worker_python.sh'] |
| 454 | |
| 455 | def worker_port_offset(self): |
| 456 | return 500 |
| 457 | |
| 458 | def scenarios(self): |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 459 | yield _ping_pong_scenario( |
| 460 | 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', |
| 461 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 462 | use_generic_payload=True, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 463 | categories=[SMOKETEST, SCALABLE]) |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 464 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 465 | yield _ping_pong_scenario( |
Ken Payson | 571c12e | 2016-06-06 23:13:27 -0700 | [diff] [blame] | 466 | 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 467 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 468 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 469 | yield _ping_pong_scenario( |
| 470 | 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 471 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 472 | |
| 473 | yield _ping_pong_scenario( |
| 474 | 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 475 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 476 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 477 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 478 | yield _ping_pong_scenario( |
| 479 | 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 480 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 481 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 482 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 483 | yield _ping_pong_scenario( |
Ken Payson | 571c12e | 2016-06-06 23:13:27 -0700 | [diff] [blame] | 484 | 'python_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 485 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Ken Payson | 9a36e6c | 2016-06-07 17:49:03 -0700 | [diff] [blame] | 486 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 487 | |
| 488 | yield _ping_pong_scenario( |
| 489 | 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 490 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 491 | server_language='c++', async_server_threads=1, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 492 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 493 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 494 | yield _ping_pong_scenario( |
| 495 | 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 496 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 497 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 498 | |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 499 | def __str__(self): |
| 500 | return 'python' |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 501 | |
Jan Tattermusch | a5780e1 | 2016-04-15 16:14:42 -0700 | [diff] [blame] | 502 | class RubyLanguage: |
| 503 | |
| 504 | def __init__(self): |
| 505 | pass |
| 506 | self.safename = str(self) |
| 507 | |
| 508 | def worker_cmdline(self): |
| 509 | return ['tools/run_tests/performance/run_worker_ruby.sh'] |
| 510 | |
| 511 | def worker_port_offset(self): |
| 512 | return 300 |
| 513 | |
| 514 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 515 | yield _ping_pong_scenario( |
| 516 | 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 517 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 518 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 519 | |
| 520 | yield _ping_pong_scenario( |
| 521 | 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 522 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 523 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 524 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 525 | yield _ping_pong_scenario( |
| 526 | 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', |
| 527 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 528 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 529 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 530 | yield _ping_pong_scenario( |
| 531 | 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', |
| 532 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 533 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 534 | |
| 535 | yield _ping_pong_scenario( |
| 536 | 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
| 537 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 538 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 539 | |
| 540 | yield _ping_pong_scenario( |
| 541 | 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
| 542 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 543 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | a5780e1 | 2016-04-15 16:14:42 -0700 | [diff] [blame] | 544 | |
| 545 | def __str__(self): |
| 546 | return 'ruby' |
| 547 | |
| 548 | |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 549 | class JavaLanguage: |
| 550 | |
| 551 | def __init__(self): |
| 552 | pass |
| 553 | self.safename = str(self) |
| 554 | |
| 555 | def worker_cmdline(self): |
| 556 | return ['tools/run_tests/performance/run_worker_java.sh'] |
| 557 | |
| 558 | def worker_port_offset(self): |
| 559 | return 400 |
| 560 | |
| 561 | def scenarios(self): |
Jan Tattermusch | e222c00 | 2016-04-20 18:55:24 -0700 | [diff] [blame] | 562 | for secure in [True, False]: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 563 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 564 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Jan Tattermusch | e222c00 | 2016-04-20 18:55:24 -0700 | [diff] [blame] | 565 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 566 | yield _ping_pong_scenario( |
| 567 | 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 568 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 569 | use_generic_payload=True, async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 570 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 571 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 572 | |
| 573 | yield _ping_pong_scenario( |
| 574 | 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 575 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 576 | async_server_threads=1, |
| 577 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 578 | |
| 579 | yield _ping_pong_scenario( |
| 580 | 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 581 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 582 | async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 583 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 584 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 585 | |
| 586 | yield _ping_pong_scenario( |
| 587 | 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 588 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 589 | async_server_threads=1, |
| 590 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 591 | |
| 592 | yield _ping_pong_scenario( |
| 593 | 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', |
| 594 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 595 | unconstrained_client='async', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 596 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 597 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 598 | |
| 599 | yield _ping_pong_scenario( |
| 600 | 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 601 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 602 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 603 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 604 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 605 | |
| 606 | yield _ping_pong_scenario( |
| 607 | 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 608 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 609 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 610 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 611 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 612 | |
| 613 | yield _ping_pong_scenario( |
| 614 | 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', |
| 615 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 616 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 617 | async_server_threads=1, |
| 618 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 619 | |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 620 | # TODO(jtattermusch): add scenarios java vs C++ |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 621 | |
| 622 | def __str__(self): |
| 623 | return 'java' |
| 624 | |
| 625 | |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 626 | class GoLanguage: |
| 627 | |
| 628 | def __init__(self): |
| 629 | pass |
| 630 | self.safename = str(self) |
| 631 | |
| 632 | def worker_cmdline(self): |
| 633 | return ['tools/run_tests/performance/run_worker_go.sh'] |
| 634 | |
| 635 | def worker_port_offset(self): |
| 636 | return 600 |
| 637 | |
| 638 | def scenarios(self): |
| 639 | for secure in [True, False]: |
| 640 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 641 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 642 | |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 643 | # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 644 | # but that's mostly because of lack of better name of the enum value. |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 645 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 646 | 'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 647 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 648 | use_generic_payload=True, async_server_threads=1, |
| 649 | secure=secure, |
| 650 | categories=smoketest_categories) |
| 651 | |
| 652 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 653 | 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 654 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 655 | async_server_threads=1, |
| 656 | secure=secure) |
| 657 | |
| 658 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 659 | 'go_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 660 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 661 | async_server_threads=1, |
| 662 | secure=secure, |
| 663 | categories=smoketest_categories) |
| 664 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 665 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 666 | yield _ping_pong_scenario( |
| 667 | 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', |
| 668 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 669 | unconstrained_client='async', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 670 | secure=secure, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 671 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 672 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 673 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 674 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 675 | 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 676 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 677 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 678 | secure=secure, |
| 679 | categories=[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 680 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 681 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 682 | # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, |
| 683 | # but that's mostly because of lack of better name of the enum value. |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 684 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 685 | 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 686 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 687 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 688 | secure=secure, |
| 689 | categories=[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 690 | |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 691 | # TODO(jtattermusch): add scenarios go vs C++ |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 692 | |
| 693 | def __str__(self): |
| 694 | return 'go' |
| 695 | |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 696 | class NodeExpressLanguage: |
| 697 | |
| 698 | def __init__(self): |
| 699 | pass |
| 700 | self.safename = str(self) |
| 701 | |
| 702 | def worker_cmdline(self): |
| 703 | return ['tools/run_tests/performance/run_worker_node.sh', |
| 704 | '--benchmark_impl=express'] |
| 705 | |
| 706 | def worker_port_offset(self): |
| 707 | return 700 |
| 708 | |
| 709 | def scenarios(self): |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 710 | yield _ping_pong_scenario( |
murgatroid99 | dabe6bd | 2016-11-09 11:03:21 -0800 | [diff] [blame] | 711 | 'node_express_json_unary_ping_pong', rpc_type='UNARY', |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 712 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 713 | categories=[SCALABLE, SMOKETEST]) |
| 714 | |
| 715 | yield _ping_pong_scenario( |
murgatroid99 | dabe6bd | 2016-11-09 11:03:21 -0800 | [diff] [blame] | 716 | 'node_express_json_async_unary_qps_unconstrained', rpc_type='UNARY', |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 717 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 718 | unconstrained_client='async', |
| 719 | categories=[SCALABLE, SMOKETEST]) |
| 720 | |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 721 | def __str__(self): |
| 722 | return 'node_express' |
| 723 | |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 724 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 725 | LANGUAGES = { |
| 726 | 'c++' : CXXLanguage(), |
| 727 | 'csharp' : CSharpLanguage(), |
| 728 | 'node' : NodeLanguage(), |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 729 | 'node_express': NodeExpressLanguage(), |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 730 | 'ruby' : RubyLanguage(), |
| 731 | 'java' : JavaLanguage(), |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 732 | 'python' : PythonLanguage(), |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 733 | 'go' : GoLanguage(), |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 734 | } |