Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 1 | # Copyright 2016 gRPC authors. |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 2 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 6 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 8 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 14 | |
| 15 | # performance scenario configuration for various languages |
| 16 | |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 17 | import math |
| 18 | |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 19 | WARMUP_SECONDS=5 |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 20 | JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 21 | BENCHMARK_SECONDS=30 |
| 22 | |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 23 | SMOKETEST='smoketest' |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 24 | SCALABLE='scalable' |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 25 | SWEEP='sweep' |
Craig Tiller | b6df247 | 2016-09-13 09:41:26 -0700 | [diff] [blame] | 26 | DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST] |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 27 | |
Jan Tattermusch | 299f97f | 2016-04-20 18:41:55 -0700 | [diff] [blame] | 28 | SECURE_SECARGS = {'use_test_ca': True, |
| 29 | 'server_host_override': 'foo.test.google.fr'} |
| 30 | |
Jan Tattermusch | 4fec48b | 2016-04-12 15:12:54 -0700 | [diff] [blame] | 31 | HISTOGRAM_PARAMS = { |
| 32 | 'resolution': 0.01, |
| 33 | 'max_possible': 60e9, |
| 34 | } |
| 35 | |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 36 | # target number of RPCs outstanding on across all client channels in |
| 37 | # non-ping-pong tests (since we can only specify per-channel numbers, the |
| 38 | # actual target will be slightly higher) |
| 39 | OUTSTANDING_REQUESTS={ |
Craig Tiller | 6c38c1b | 2016-09-13 08:34:27 -0700 | [diff] [blame] | 40 | 'async': 6400, |
Vijay Pai | fba000c | 2017-05-23 14:17:17 -0700 | [diff] [blame] | 41 | 'async-limited': 800, |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 42 | 'sync': 1000 |
| 43 | } |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 44 | |
| 45 | # wide is the number of client channels in multi-channel tests (1 otherwise) |
| 46 | WIDE=64 |
| 47 | |
| 48 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 49 | def _get_secargs(is_secure): |
| 50 | if is_secure: |
| 51 | return SECURE_SECARGS |
| 52 | else: |
| 53 | return None |
| 54 | |
| 55 | |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 56 | def remove_nonproto_fields(scenario): |
| 57 | """Remove special-purpose that contains some extra info about the scenario |
| 58 | but don't belong to the ScenarioConfig protobuf message""" |
| 59 | scenario.pop('CATEGORIES', None) |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 60 | scenario.pop('CLIENT_LANGUAGE', None) |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 61 | scenario.pop('SERVER_LANGUAGE', None) |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 62 | scenario.pop('EXCLUDED_POLL_ENGINES', None) |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 63 | return scenario |
| 64 | |
| 65 | |
Craig Tiller | c5aa700 | 2016-09-13 09:36:09 -0700 | [diff] [blame] | 66 | def geometric_progression(start, stop, step): |
| 67 | n = start |
| 68 | while n < stop: |
| 69 | yield int(round(n)) |
| 70 | n *= step |
| 71 | |
| 72 | |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 73 | def _payload_type(use_generic_payload, req_size, resp_size): |
| 74 | r = {} |
| 75 | sizes = { |
| 76 | 'req_size': req_size, |
| 77 | 'resp_size': resp_size, |
| 78 | } |
| 79 | if use_generic_payload: |
| 80 | r['bytebuf_params'] = sizes |
| 81 | else: |
| 82 | r['simple_params'] = sizes |
Alexander Polcyn | c6e333e | 2017-02-03 09:26:35 -0800 | [diff] [blame] | 83 | return r |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 84 | |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 85 | def _add_channel_arg(config, key, value): |
| 86 | if 'channel_args' in config: |
| 87 | channel_args = config['channel_args'] |
| 88 | else: |
| 89 | channel_args = [] |
| 90 | config['channel_args'] = channel_args |
| 91 | arg = {'name': key} |
| 92 | if isinstance(value, int): |
| 93 | arg['int_value'] = value |
| 94 | else: |
| 95 | arg['str_value'] = value |
| 96 | channel_args.append(arg) |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 97 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 98 | def _ping_pong_scenario(name, rpc_type, |
| 99 | client_type, server_type, |
| 100 | secure=True, |
| 101 | use_generic_payload=False, |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 102 | req_size=0, |
| 103 | resp_size=0, |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 104 | unconstrained_client=None, |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 105 | client_language=None, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 106 | server_language=None, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 107 | async_server_threads=0, |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 108 | server_threads_per_cq=0, |
| 109 | client_threads_per_cq=0, |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 110 | warmup_seconds=WARMUP_SECONDS, |
Craig Tiller | b6df247 | 2016-09-13 09:41:26 -0700 | [diff] [blame] | 111 | categories=DEFAULT_CATEGORIES, |
Craig Tiller | c5aa700 | 2016-09-13 09:36:09 -0700 | [diff] [blame] | 112 | channels=None, |
Craig Tiller | 0c80c7d | 2016-09-28 16:30:55 -0700 | [diff] [blame] | 113 | outstanding=None, |
Vijay Pai | f6f28a7 | 2017-05-08 15:13:32 -0700 | [diff] [blame] | 114 | num_clients=None, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 115 | resource_quota_size=None, |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 116 | messages_per_stream=None, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 117 | excluded_poll_engines=[], |
| 118 | minimal_stack=False): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 119 | """Creates a basic ping pong scenario.""" |
| 120 | scenario = { |
| 121 | 'name': name, |
| 122 | 'num_servers': 1, |
| 123 | 'num_clients': 1, |
| 124 | 'client_config': { |
| 125 | 'client_type': client_type, |
| 126 | 'security_params': _get_secargs(secure), |
| 127 | 'outstanding_rpcs_per_channel': 1, |
| 128 | 'client_channels': 1, |
| 129 | 'async_client_threads': 1, |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 130 | 'threads_per_cq': client_threads_per_cq, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 131 | 'rpc_type': rpc_type, |
| 132 | 'load_params': { |
| 133 | 'closed_loop': {} |
| 134 | }, |
| 135 | 'histogram_params': HISTOGRAM_PARAMS, |
Craig Tiller | f1633a9 | 2017-06-19 10:01:52 -0700 | [diff] [blame] | 136 | 'channel_args': [], |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 137 | }, |
| 138 | 'server_config': { |
| 139 | 'server_type': server_type, |
| 140 | 'security_params': _get_secargs(secure), |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 141 | 'async_server_threads': async_server_threads, |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 142 | 'threads_per_cq': server_threads_per_cq, |
Craig Tiller | f1633a9 | 2017-06-19 10:01:52 -0700 | [diff] [blame] | 143 | 'channel_args': [], |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 144 | }, |
| 145 | 'warmup_seconds': warmup_seconds, |
| 146 | 'benchmark_seconds': BENCHMARK_SECONDS |
| 147 | } |
Craig Tiller | 20afa3d | 2016-10-17 14:52:14 -0700 | [diff] [blame] | 148 | if resource_quota_size: |
| 149 | scenario['server_config']['resource_quota_size'] = resource_quota_size |
Jan Tattermusch | 28f6c01 | 2016-05-22 21:23:43 -0400 | [diff] [blame] | 150 | if use_generic_payload: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 151 | if server_type != 'ASYNC_GENERIC_SERVER': |
| 152 | raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') |
Alexander Polcyn | f797c65 | 2017-02-08 14:29:30 -0800 | [diff] [blame] | 153 | scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) |
| 154 | |
Craig Tiller | 412fa2a | 2017-01-18 15:21:54 -0800 | [diff] [blame] | 155 | scenario['client_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 156 | |
Sree Kuchibhotla | 0fda880 | 2017-08-30 20:34:51 -0700 | [diff] [blame] | 157 | # Optimization target of 'throughput' does not work well with epoll1 polling |
| 158 | # engine. Use the default value of 'blend' |
Craig Tiller | ebec0db | 2017-09-07 12:47:35 -0700 | [diff] [blame] | 159 | optimization_target = 'throughput' |
Craig Tiller | f1633a9 | 2017-06-19 10:01:52 -0700 | [diff] [blame] | 160 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 161 | if unconstrained_client: |
Craig Tiller | 0c98675 | 2016-09-13 10:01:26 -0700 | [diff] [blame] | 162 | outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] |
Craig Tiller | faa835c | 2017-01-18 15:28:44 -0800 | [diff] [blame] | 163 | # clamp buffer usage to something reasonable (16 gig for now) |
| 164 | MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 |
| 165 | if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: |
| 166 | outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, resp_size)) |
Craig Tiller | 15372a3 | 2016-09-08 09:21:55 -0700 | [diff] [blame] | 167 | wide = channels if channels is not None else WIDE |
Craig Tiller | 0c98675 | 2016-09-13 10:01:26 -0700 | [diff] [blame] | 168 | deep = int(math.ceil(1.0 * outstanding_calls / wide)) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 169 | |
Vijay Pai | f6f28a7 | 2017-05-08 15:13:32 -0700 | [diff] [blame] | 170 | scenario['num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available. |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 171 | scenario['client_config']['outstanding_rpcs_per_channel'] = deep |
| 172 | scenario['client_config']['client_channels'] = wide |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 173 | scenario['client_config']['async_client_threads'] = 0 |
| 174 | else: |
| 175 | scenario['client_config']['outstanding_rpcs_per_channel'] = 1 |
| 176 | scenario['client_config']['client_channels'] = 1 |
| 177 | scenario['client_config']['async_client_threads'] = 1 |
Craig Tiller | ebec0db | 2017-09-07 12:47:35 -0700 | [diff] [blame] | 178 | optimization_target = 'latency' |
Craig Tiller | f1633a9 | 2017-06-19 10:01:52 -0700 | [diff] [blame] | 179 | |
| 180 | optimization_channel_arg = { |
| 181 | 'name': 'grpc.optimization_target', |
| 182 | 'str_value': optimization_target |
| 183 | } |
| 184 | scenario['client_config']['channel_args'].append(optimization_channel_arg) |
| 185 | scenario['server_config']['channel_args'].append(optimization_channel_arg) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 186 | |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 187 | if minimal_stack: |
| 188 | _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1) |
| 189 | _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1) |
| 190 | |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 191 | if messages_per_stream: |
| 192 | scenario['client_config']['messages_per_stream'] = messages_per_stream |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 193 | if client_language: |
| 194 | # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py |
| 195 | scenario['CLIENT_LANGUAGE'] = client_language |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 196 | if server_language: |
| 197 | # the SERVER_LANGUAGE field is recognized by run_performance_tests.py |
| 198 | scenario['SERVER_LANGUAGE'] = server_language |
Jan Tattermusch | a21c7e9 | 2016-05-05 17:31:52 -0700 | [diff] [blame] | 199 | if categories: |
| 200 | scenario['CATEGORIES'] = categories |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 201 | if len(excluded_poll_engines): |
| 202 | # The polling engines for which this scenario is excluded |
| 203 | scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 204 | return scenario |
| 205 | |
| 206 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 207 | class CXXLanguage: |
| 208 | |
| 209 | def __init__(self): |
| 210 | self.safename = 'cxx' |
| 211 | |
| 212 | def worker_cmdline(self): |
| 213 | return ['bins/opt/qps_worker'] |
| 214 | |
| 215 | def worker_port_offset(self): |
| 216 | return 0 |
| 217 | |
| 218 | def scenarios(self): |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 219 | # TODO(ctiller): add 70% load latency test |
Vijay Pai | f6f28a7 | 2017-05-08 15:13:32 -0700 | [diff] [blame] | 220 | yield _ping_pong_scenario( |
| 221 | 'cpp_protobuf_async_unary_1channel_100rpcs_1MB', rpc_type='UNARY', |
| 222 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 223 | req_size=1024*1024, resp_size=1024*1024, |
| 224 | unconstrained_client='async', outstanding=100, channels=1, |
| 225 | num_clients=1, |
| 226 | secure=False, |
| 227 | categories=[SMOKETEST] + [SCALABLE]) |
| 228 | |
| 229 | yield _ping_pong_scenario( |
| 230 | 'cpp_protobuf_async_streaming_from_client_1channel_1MB', rpc_type='STREAMING_FROM_CLIENT', |
| 231 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 232 | req_size=1024*1024, resp_size=1024*1024, |
| 233 | unconstrained_client='async', outstanding=1, channels=1, |
| 234 | num_clients=1, |
| 235 | secure=False, |
| 236 | categories=[SMOKETEST] + [SCALABLE]) |
| 237 | |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 238 | for secure in [True, False]: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 239 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 240 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Craig Tiller | 0bda0b3 | 2016-03-03 12:51:53 -0800 | [diff] [blame] | 241 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 242 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 243 | 'cpp_generic_async_streaming_ping_pong_%s' % secstr, |
| 244 | rpc_type='STREAMING', |
| 245 | client_type='ASYNC_CLIENT', |
| 246 | server_type='ASYNC_GENERIC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 247 | use_generic_payload=True, async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 248 | secure=secure, |
| 249 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 250 | |
| 251 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 252 | 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, |
| 253 | rpc_type='STREAMING', |
| 254 | client_type='ASYNC_CLIENT', |
| 255 | server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 256 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 257 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 258 | minimal_stack=not secure, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 259 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 260 | |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 261 | for mps in geometric_progression(1, 20, 10): |
| 262 | yield _ping_pong_scenario( |
| 263 | 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr), |
| 264 | rpc_type='STREAMING', |
| 265 | client_type='ASYNC_CLIENT', |
| 266 | server_type='ASYNC_GENERIC_SERVER', |
| 267 | unconstrained_client='async', use_generic_payload=True, |
| 268 | secure=secure, messages_per_stream=mps, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 269 | minimal_stack=not secure, |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 270 | categories=smoketest_categories+[SCALABLE]) |
| 271 | |
| 272 | for mps in geometric_progression(1, 200, math.sqrt(10)): |
| 273 | yield _ping_pong_scenario( |
| 274 | 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr), |
| 275 | rpc_type='STREAMING', |
| 276 | client_type='ASYNC_CLIENT', |
| 277 | server_type='ASYNC_GENERIC_SERVER', |
| 278 | unconstrained_client='async', use_generic_payload=True, |
| 279 | secure=secure, messages_per_stream=mps, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 280 | minimal_stack=not secure, |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 281 | categories=[SWEEP]) |
| 282 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 283 | yield _ping_pong_scenario( |
Vijay Pai | 0522251 | 2017-02-21 23:52:29 -0800 | [diff] [blame] | 284 | 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, |
| 285 | rpc_type='STREAMING', |
| 286 | req_size=1024*1024, |
| 287 | resp_size=1024*1024, |
| 288 | client_type='ASYNC_CLIENT', |
| 289 | server_type='ASYNC_GENERIC_SERVER', |
| 290 | unconstrained_client='async', use_generic_payload=True, |
| 291 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 292 | minimal_stack=not secure, |
Vijay Pai | 0522251 | 2017-02-21 23:52:29 -0800 | [diff] [blame] | 293 | categories=smoketest_categories+[SCALABLE], |
| 294 | channels=1, outstanding=100) |
| 295 | |
| 296 | yield _ping_pong_scenario( |
| 297 | 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % secstr, |
| 298 | rpc_type='STREAMING', |
| 299 | req_size=64*1024, |
| 300 | resp_size=64*1024, |
| 301 | client_type='ASYNC_CLIENT', |
| 302 | server_type='ASYNC_GENERIC_SERVER', |
| 303 | unconstrained_client='async', use_generic_payload=True, |
| 304 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 305 | minimal_stack=not secure, |
Vijay Pai | 0522251 | 2017-02-21 23:52:29 -0800 | [diff] [blame] | 306 | categories=smoketest_categories+[SCALABLE]) |
| 307 | |
Ken Payson | 4536138 | 2017-06-14 15:12:52 -0700 | [diff] [blame] | 308 | # TODO(https://github.com/grpc/grpc/issues/11500) Re-enable this test |
| 309 | #yield _ping_pong_scenario( |
| 310 | # 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, |
| 311 | # rpc_type='STREAMING', |
| 312 | # client_type='ASYNC_CLIENT', |
| 313 | # server_type='ASYNC_GENERIC_SERVER', |
| 314 | # unconstrained_client='async-limited', use_generic_payload=True, |
| 315 | # secure=secure, |
| 316 | # client_threads_per_cq=1000000, server_threads_per_cq=1000000, |
| 317 | # categories=smoketest_categories+[SCALABLE]) |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 318 | |
| 319 | yield _ping_pong_scenario( |
| 320 | 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr, |
| 321 | rpc_type='STREAMING', |
| 322 | client_type='ASYNC_CLIENT', |
| 323 | server_type='ASYNC_GENERIC_SERVER', |
| 324 | unconstrained_client='async', use_generic_payload=True, |
| 325 | secure=secure, |
| 326 | client_threads_per_cq=2, server_threads_per_cq=2, |
| 327 | categories=smoketest_categories+[SCALABLE]) |
| 328 | |
Ken Payson | 4536138 | 2017-06-14 15:12:52 -0700 | [diff] [blame] | 329 | #yield _ping_pong_scenario( |
| 330 | # 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % secstr, |
| 331 | # rpc_type='STREAMING', |
| 332 | # client_type='ASYNC_CLIENT', |
| 333 | # server_type='ASYNC_SERVER', |
| 334 | # unconstrained_client='async-limited', |
| 335 | # secure=secure, |
| 336 | # client_threads_per_cq=1000000, server_threads_per_cq=1000000, |
| 337 | # categories=smoketest_categories+[SCALABLE]) |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 338 | |
| 339 | yield _ping_pong_scenario( |
| 340 | 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr, |
| 341 | rpc_type='STREAMING', |
| 342 | client_type='ASYNC_CLIENT', |
| 343 | server_type='ASYNC_SERVER', |
| 344 | unconstrained_client='async', |
| 345 | secure=secure, |
| 346 | client_threads_per_cq=2, server_threads_per_cq=2, |
| 347 | categories=smoketest_categories+[SCALABLE]) |
| 348 | |
Ken Payson | 4536138 | 2017-06-14 15:12:52 -0700 | [diff] [blame] | 349 | #yield _ping_pong_scenario( |
| 350 | # 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, |
| 351 | # rpc_type='UNARY', |
| 352 | # client_type='ASYNC_CLIENT', |
| 353 | # server_type='ASYNC_SERVER', |
| 354 | # unconstrained_client='async-limited', |
| 355 | # secure=secure, |
| 356 | # client_threads_per_cq=1000000, server_threads_per_cq=1000000, |
| 357 | # categories=smoketest_categories+[SCALABLE]) |
Vijay Pai | 4b07aab | 2017-02-22 15:30:16 -0800 | [diff] [blame] | 358 | |
| 359 | yield _ping_pong_scenario( |
| 360 | 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % secstr, |
| 361 | rpc_type='UNARY', |
| 362 | client_type='ASYNC_CLIENT', |
| 363 | server_type='ASYNC_SERVER', |
| 364 | unconstrained_client='async', |
| 365 | secure=secure, |
| 366 | client_threads_per_cq=2, server_threads_per_cq=2, |
| 367 | categories=smoketest_categories+[SCALABLE]) |
| 368 | |
Vijay Pai | 0522251 | 2017-02-21 23:52:29 -0800 | [diff] [blame] | 369 | yield _ping_pong_scenario( |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 370 | 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, |
| 371 | rpc_type='STREAMING', |
| 372 | client_type='ASYNC_CLIENT', |
| 373 | server_type='ASYNC_GENERIC_SERVER', |
Vijay Pai | fba000c | 2017-05-23 14:17:17 -0700 | [diff] [blame] | 374 | unconstrained_client='async-limited', use_generic_payload=True, |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 375 | async_server_threads=1, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 376 | minimal_stack=not secure, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 377 | secure=secure) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 378 | |
Sree Kuchibhotla | 923d641 | 2016-09-26 19:32:17 -0700 | [diff] [blame] | 379 | yield _ping_pong_scenario( |
| 380 | 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' % |
| 381 | (secstr), |
| 382 | rpc_type='UNARY', |
| 383 | client_type='ASYNC_CLIENT', |
| 384 | server_type='SYNC_SERVER', |
| 385 | unconstrained_client='async', |
| 386 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 387 | minimal_stack=not secure, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 388 | categories=smoketest_categories + [SCALABLE], |
| 389 | excluded_poll_engines = ['poll-cv']) |
Sree Kuchibhotla | 923d641 | 2016-09-26 19:32:17 -0700 | [diff] [blame] | 390 | |
Sree Kuchibhotla | a83ee60 | 2016-10-25 11:54:52 -0700 | [diff] [blame] | 391 | yield _ping_pong_scenario( |
Vijay Pai | 47fc606 | 2017-03-31 11:03:16 -0700 | [diff] [blame] | 392 | 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' % |
| 393 | (secstr), |
| 394 | rpc_type='UNARY', |
| 395 | client_type='ASYNC_CLIENT', |
| 396 | server_type='ASYNC_SERVER', |
| 397 | channels=1, |
| 398 | outstanding=64, |
| 399 | req_size=128, |
| 400 | resp_size=8*1024*1024, |
| 401 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 402 | minimal_stack=not secure, |
Vijay Pai | 47fc606 | 2017-03-31 11:03:16 -0700 | [diff] [blame] | 403 | categories=smoketest_categories + [SCALABLE]) |
| 404 | |
| 405 | yield _ping_pong_scenario( |
Sree Kuchibhotla | a83ee60 | 2016-10-25 11:54:52 -0700 | [diff] [blame] | 406 | 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' % secstr, |
| 407 | rpc_type='STREAMING', |
| 408 | client_type='ASYNC_CLIENT', |
| 409 | server_type='SYNC_SERVER', |
| 410 | unconstrained_client='async', |
| 411 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 412 | minimal_stack=not secure, |
Sree Kuchibhotla | 70d9ca4 | 2017-01-27 10:54:05 -0800 | [diff] [blame] | 413 | categories=smoketest_categories+[SCALABLE], |
| 414 | excluded_poll_engines = ['poll-cv']) |
Sree Kuchibhotla | a83ee60 | 2016-10-25 11:54:52 -0700 | [diff] [blame] | 415 | |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 416 | yield _ping_pong_scenario( |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 417 | 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 418 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 419 | req_size=1024*1024, resp_size=1024*1024, |
| 420 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 421 | minimal_stack=not secure, |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 422 | categories=smoketest_categories + [SCALABLE]) |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 423 | |
Vijay Pai | eea8cf0 | 2017-03-23 16:19:00 -0700 | [diff] [blame] | 424 | for rpc_type in ['unary', 'streaming', 'streaming_from_client', 'streaming_from_server']: |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 425 | for synchronicity in ['sync', 'async']: |
| 426 | yield _ping_pong_scenario( |
| 427 | 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr), |
| 428 | rpc_type=rpc_type.upper(), |
| 429 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 430 | server_type='%s_SERVER' % synchronicity.upper(), |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 431 | async_server_threads=1, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 432 | minimal_stack=not secure, |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 433 | secure=secure) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 434 | |
Craig Tiller | faa835c | 2017-01-18 15:28:44 -0800 | [diff] [blame] | 435 | for size in geometric_progression(1, 1024*1024*1024+1, 8): |
| 436 | yield _ping_pong_scenario( |
| 437 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % (synchronicity, rpc_type, secstr, size), |
| 438 | rpc_type=rpc_type.upper(), |
| 439 | req_size=size, |
| 440 | resp_size=size, |
| 441 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 442 | server_type='%s_SERVER' % synchronicity.upper(), |
| 443 | unconstrained_client=synchronicity, |
| 444 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 445 | minimal_stack=not secure, |
Craig Tiller | faa835c | 2017-01-18 15:28:44 -0800 | [diff] [blame] | 446 | categories=[SWEEP]) |
| 447 | |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 448 | yield _ping_pong_scenario( |
| 449 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr), |
| 450 | rpc_type=rpc_type.upper(), |
| 451 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 452 | server_type='%s_SERVER' % synchronicity.upper(), |
| 453 | unconstrained_client=synchronicity, |
| 454 | secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 455 | minimal_stack=not secure, |
Craig Tiller | 2426163 | 2017-09-08 16:12:05 -0700 | [diff] [blame] | 456 | server_threads_per_cq=3, |
| 457 | client_threads_per_cq=3, |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 458 | categories=smoketest_categories+[SCALABLE]) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 459 | |
Vijay Pai | a582def | 2017-03-09 05:47:46 -0800 | [diff] [blame] | 460 | # TODO(vjpai): Re-enable this test. It has a lot of timeouts |
| 461 | # and hasn't yet been conclusively identified as a test failure |
| 462 | # or race in the library |
| 463 | # yield _ping_pong_scenario( |
| 464 | # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), |
| 465 | # rpc_type=rpc_type.upper(), |
| 466 | # client_type='%s_CLIENT' % synchronicity.upper(), |
| 467 | # server_type='%s_SERVER' % synchronicity.upper(), |
| 468 | # unconstrained_client=synchronicity, |
| 469 | # secure=secure, |
| 470 | # categories=smoketest_categories+[SCALABLE], |
| 471 | # resource_quota_size=500*1024) |
Craig Tiller | 0c80c7d | 2016-09-28 16:30:55 -0700 | [diff] [blame] | 472 | |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 473 | if rpc_type == 'streaming': |
| 474 | for mps in geometric_progression(1, 20, 10): |
| 475 | yield _ping_pong_scenario( |
| 476 | 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr), |
| 477 | rpc_type=rpc_type.upper(), |
| 478 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 479 | server_type='%s_SERVER' % synchronicity.upper(), |
| 480 | unconstrained_client=synchronicity, |
| 481 | secure=secure, messages_per_stream=mps, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 482 | minimal_stack=not secure, |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 483 | categories=smoketest_categories+[SCALABLE]) |
| 484 | |
| 485 | for mps in geometric_progression(1, 200, math.sqrt(10)): |
| 486 | yield _ping_pong_scenario( |
| 487 | 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr), |
| 488 | rpc_type=rpc_type.upper(), |
| 489 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 490 | server_type='%s_SERVER' % synchronicity.upper(), |
| 491 | unconstrained_client=synchronicity, |
| 492 | secure=secure, messages_per_stream=mps, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 493 | minimal_stack=not secure, |
Vijay Pai | 45a9aba | 2017-02-27 13:30:37 -0800 | [diff] [blame] | 494 | categories=[SWEEP]) |
| 495 | |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 496 | for channels in geometric_progression(1, 20000, math.sqrt(10)): |
| 497 | for outstanding in geometric_progression(1, 200000, math.sqrt(10)): |
| 498 | if synchronicity == 'sync' and outstanding > 1200: continue |
| 499 | if outstanding < channels: continue |
| 500 | yield _ping_pong_scenario( |
| 501 | 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding), |
| 502 | rpc_type=rpc_type.upper(), |
| 503 | client_type='%s_CLIENT' % synchronicity.upper(), |
| 504 | server_type='%s_SERVER' % synchronicity.upper(), |
| 505 | unconstrained_client=synchronicity, secure=secure, |
Craig Tiller | 277b1fa | 2017-04-26 08:31:47 -0700 | [diff] [blame] | 506 | minimal_stack=not secure, |
Craig Tiller | 601cff4 | 2016-09-19 07:30:45 -0700 | [diff] [blame] | 507 | categories=[SWEEP], channels=channels, outstanding=outstanding) |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 508 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 509 | def __str__(self): |
| 510 | return 'c++' |
| 511 | |
| 512 | |
| 513 | class CSharpLanguage: |
| 514 | |
| 515 | def __init__(self): |
| 516 | self.safename = str(self) |
| 517 | |
| 518 | def worker_cmdline(self): |
| 519 | return ['tools/run_tests/performance/run_worker_csharp.sh'] |
| 520 | |
| 521 | def worker_port_offset(self): |
| 522 | return 100 |
| 523 | |
| 524 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 525 | yield _ping_pong_scenario( |
| 526 | 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', |
| 527 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 528 | use_generic_payload=True, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 529 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 530 | |
| 531 | yield _ping_pong_scenario( |
Jan Tattermusch | aba4ee2 | 2017-04-25 10:55:54 +0200 | [diff] [blame] | 532 | 'csharp_generic_async_streaming_ping_pong_insecure_1MB', rpc_type='STREAMING', |
| 533 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 534 | req_size=1024*1024, resp_size=1024*1024, |
| 535 | use_generic_payload=True, |
| 536 | secure=False, |
| 537 | categories=[SMOKETEST, SCALABLE]) |
| 538 | |
| 539 | yield _ping_pong_scenario( |
| 540 | 'csharp_generic_async_streaming_qps_unconstrained_insecure', rpc_type='STREAMING', |
| 541 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 542 | unconstrained_client='async', use_generic_payload=True, |
| 543 | secure=False, |
| 544 | categories=[SMOKETEST, SCALABLE]) |
| 545 | |
| 546 | yield _ping_pong_scenario( |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 547 | 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 548 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
| 549 | |
| 550 | yield _ping_pong_scenario( |
| 551 | 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 552 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 553 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 554 | |
| 555 | yield _ping_pong_scenario( |
| 556 | 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', |
| 557 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') |
| 558 | |
Jan Tattermusch | 16713b9 | 2016-05-09 14:34:11 -0700 | [diff] [blame] | 559 | yield _ping_pong_scenario( |
| 560 | 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 561 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 562 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 563 | categories=[SMOKETEST,SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 564 | |
Jan Tattermusch | 16713b9 | 2016-05-09 14:34:11 -0700 | [diff] [blame] | 565 | yield _ping_pong_scenario( |
| 566 | 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', |
| 567 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 568 | unconstrained_client='async', |
| 569 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 570 | |
Jan Tattermusch | abd1b38 | 2016-05-08 18:47:40 -0700 | [diff] [blame] | 571 | yield _ping_pong_scenario( |
| 572 | 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
| 573 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 574 | server_language='c++', async_server_threads=1, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 575 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 576 | |
| 577 | yield _ping_pong_scenario( |
| 578 | 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 579 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 580 | server_language='c++', async_server_threads=1) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 581 | |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 582 | yield _ping_pong_scenario( |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 583 | 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 584 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 585 | unconstrained_client='async', server_language='c++', |
| 586 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 587 | |
| 588 | yield _ping_pong_scenario( |
| 589 | 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 590 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 591 | unconstrained_client='sync', server_language='c++', |
| 592 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 593 | |
| 594 | yield _ping_pong_scenario( |
Jan Tattermusch | 37a907e | 2016-05-13 13:49:43 -0700 | [diff] [blame] | 595 | 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', |
| 596 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 597 | unconstrained_client='async', client_language='c++', |
| 598 | categories=[SCALABLE]) |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 599 | |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 600 | yield _ping_pong_scenario( |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 601 | 'csharp_protobuf_async_unary_ping_pong_1MB', rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 602 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 603 | req_size=1024*1024, resp_size=1024*1024, |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 604 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 605 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 606 | def __str__(self): |
| 607 | return 'csharp' |
| 608 | |
| 609 | |
| 610 | class NodeLanguage: |
| 611 | |
| 612 | def __init__(self): |
| 613 | pass |
| 614 | self.safename = str(self) |
| 615 | |
| 616 | def worker_cmdline(self): |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 617 | return ['tools/run_tests/performance/run_worker_node.sh', |
| 618 | '--benchmark_impl=grpc'] |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 619 | |
| 620 | def worker_port_offset(self): |
| 621 | return 200 |
| 622 | |
| 623 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 624 | # TODO(jtattermusch): make this scenario work |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 625 | yield _ping_pong_scenario( |
| 626 | 'node_generic_streaming_ping_pong', rpc_type='STREAMING', |
| 627 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 628 | use_generic_payload=True) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 629 | |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 630 | yield _ping_pong_scenario( |
| 631 | 'node_protobuf_streaming_ping_pong', rpc_type='STREAMING', |
| 632 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 633 | |
| 634 | yield _ping_pong_scenario( |
| 635 | 'node_protobuf_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 636 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 637 | categories=[SCALABLE, SMOKETEST]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 638 | |
| 639 | yield _ping_pong_scenario( |
murgatroid99 | 9030c81 | 2016-09-16 13:25:08 -0700 | [diff] [blame] | 640 | 'cpp_to_node_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 641 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
murgatroid99 | 9030c81 | 2016-09-16 13:25:08 -0700 | [diff] [blame] | 642 | client_language='c++') |
| 643 | |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 644 | yield _ping_pong_scenario( |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 645 | 'node_protobuf_unary_ping_pong_1MB', rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 646 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 647 | req_size=1024*1024, resp_size=1024*1024, |
murgatroid99 | bf33410 | 2017-04-18 11:11:28 -0700 | [diff] [blame] | 648 | categories=[SCALABLE]) |
| 649 | |
| 650 | sizes = [('1B', 1), ('1KB', 1024), ('10KB', 10 * 1024), |
| 651 | ('1MB', 1024 * 1024), ('10MB', 10 * 1024 * 1024), |
| 652 | ('100MB', 100 * 1024 * 1024)] |
| 653 | |
| 654 | for size_name, size in sizes: |
| 655 | for secure in (True, False): |
| 656 | yield _ping_pong_scenario( |
| 657 | 'node_protobuf_unary_ping_pong_%s_resp_%s' % |
| 658 | (size_name, 'secure' if secure else 'insecure'), |
| 659 | rpc_type='UNARY', |
| 660 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 661 | req_size=0, resp_size=size, |
| 662 | secure=secure, |
| 663 | categories=[SCALABLE]) |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 664 | |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 665 | yield _ping_pong_scenario( |
| 666 | 'node_protobuf_unary_qps_unconstrained', rpc_type='UNARY', |
| 667 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 668 | unconstrained_client='async', |
| 669 | categories=[SCALABLE, SMOKETEST]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 670 | |
murgatroid99 | bf33410 | 2017-04-18 11:11:28 -0700 | [diff] [blame] | 671 | yield _ping_pong_scenario( |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 672 | 'node_protobuf_streaming_qps_unconstrained', rpc_type='STREAMING', |
murgatroid99 | bf33410 | 2017-04-18 11:11:28 -0700 | [diff] [blame] | 673 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 674 | unconstrained_client='async') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 675 | |
murgatroid99 | bf33410 | 2017-04-18 11:11:28 -0700 | [diff] [blame] | 676 | yield _ping_pong_scenario( |
| 677 | 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
| 678 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 679 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 680 | |
murgatroid99 | 648832f | 2017-05-23 17:20:38 -0700 | [diff] [blame] | 681 | yield _ping_pong_scenario( |
| 682 | 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', |
| 683 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 684 | server_language='c++', async_server_threads=1) |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 685 | |
| 686 | def __str__(self): |
| 687 | return 'node' |
| 688 | |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 689 | class PythonLanguage: |
| 690 | |
| 691 | def __init__(self): |
| 692 | self.safename = 'python' |
| 693 | |
| 694 | def worker_cmdline(self): |
| 695 | return ['tools/run_tests/performance/run_worker_python.sh'] |
| 696 | |
| 697 | def worker_port_offset(self): |
| 698 | return 500 |
| 699 | |
| 700 | def scenarios(self): |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 701 | yield _ping_pong_scenario( |
| 702 | 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', |
| 703 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 704 | use_generic_payload=True, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 705 | categories=[SMOKETEST, SCALABLE]) |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 706 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 707 | yield _ping_pong_scenario( |
Ken Payson | 571c12e | 2016-06-06 23:13:27 -0700 | [diff] [blame] | 708 | 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 709 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 710 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 711 | yield _ping_pong_scenario( |
| 712 | 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 713 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 714 | |
| 715 | yield _ping_pong_scenario( |
| 716 | 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 717 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 718 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 719 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 720 | yield _ping_pong_scenario( |
| 721 | 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 722 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 723 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 724 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 725 | yield _ping_pong_scenario( |
Ken Payson | 571c12e | 2016-06-06 23:13:27 -0700 | [diff] [blame] | 726 | 'python_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 727 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Ken Payson | 9a36e6c | 2016-06-07 17:49:03 -0700 | [diff] [blame] | 728 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 729 | |
| 730 | yield _ping_pong_scenario( |
| 731 | 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 732 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 733 | server_language='c++', async_server_threads=1, |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 734 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 735 | |
Ken Payson | 1eb8d54 | 2016-05-10 16:57:14 -0700 | [diff] [blame] | 736 | yield _ping_pong_scenario( |
| 737 | 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Ken Payson | 45c0f2b | 2016-07-06 19:26:09 -0700 | [diff] [blame] | 738 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 739 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 740 | |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 741 | yield _ping_pong_scenario( |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 742 | 'python_protobuf_sync_unary_ping_pong_1MB', rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 743 | client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', |
| 744 | req_size=1024*1024, resp_size=1024*1024, |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 745 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 746 | |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 747 | def __str__(self): |
| 748 | return 'python' |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 749 | |
Jan Tattermusch | a5780e1 | 2016-04-15 16:14:42 -0700 | [diff] [blame] | 750 | class RubyLanguage: |
| 751 | |
| 752 | def __init__(self): |
| 753 | pass |
| 754 | self.safename = str(self) |
| 755 | |
| 756 | def worker_cmdline(self): |
| 757 | return ['tools/run_tests/performance/run_worker_ruby.sh'] |
| 758 | |
| 759 | def worker_port_offset(self): |
| 760 | return 300 |
| 761 | |
| 762 | def scenarios(self): |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 763 | yield _ping_pong_scenario( |
| 764 | 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 765 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 766 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 767 | |
| 768 | yield _ping_pong_scenario( |
| 769 | 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 770 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 771 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 772 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 773 | yield _ping_pong_scenario( |
| 774 | 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', |
| 775 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 776 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 777 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 778 | yield _ping_pong_scenario( |
| 779 | 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', |
| 780 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 781 | unconstrained_client='sync') |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 782 | |
| 783 | yield _ping_pong_scenario( |
| 784 | 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', |
| 785 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 786 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 787 | |
| 788 | yield _ping_pong_scenario( |
| 789 | 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', |
| 790 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Alexander Polcyn | a6e796f | 2017-01-11 13:49:43 -0800 | [diff] [blame] | 791 | server_language='c++', async_server_threads=1) |
Jan Tattermusch | a5780e1 | 2016-04-15 16:14:42 -0700 | [diff] [blame] | 792 | |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 793 | yield _ping_pong_scenario( |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 794 | 'ruby_protobuf_unary_ping_pong_1MB', rpc_type='UNARY', |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 795 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 796 | req_size=1024*1024, resp_size=1024*1024, |
Jan Tattermusch | 58aa2cb | 2017-04-07 09:30:49 +0200 | [diff] [blame] | 797 | categories=[SMOKETEST, SCALABLE]) |
Jan Tattermusch | 46df13b | 2017-03-30 17:57:06 +0200 | [diff] [blame] | 798 | |
Jan Tattermusch | a5780e1 | 2016-04-15 16:14:42 -0700 | [diff] [blame] | 799 | def __str__(self): |
| 800 | return 'ruby' |
| 801 | |
| 802 | |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 803 | class PhpLanguage: |
| 804 | |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 805 | def __init__(self, use_protobuf_c_extension=False): |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 806 | pass |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 807 | self.use_protobuf_c_extension=use_protobuf_c_extension |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 808 | self.safename = str(self) |
| 809 | |
| 810 | def worker_cmdline(self): |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 811 | if self.use_protobuf_c_extension: |
| 812 | return ['tools/run_tests/performance/run_worker_php.sh -c'] |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 813 | return ['tools/run_tests/performance/run_worker_php.sh'] |
| 814 | |
| 815 | def worker_port_offset(self): |
| 816 | return 800 |
| 817 | |
| 818 | def scenarios(self): |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 819 | php_extension_mode='php_protobuf_php_extension' |
| 820 | if self.use_protobuf_c_extension: |
| 821 | php_extension_mode='php_protobuf_c_extension' |
| 822 | |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 823 | yield _ping_pong_scenario( |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 824 | '%s_to_cpp_protobuf_sync_unary_ping_pong' % php_extension_mode, |
| 825 | rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 826 | server_language='c++', async_server_threads=1) |
| 827 | |
| 828 | yield _ping_pong_scenario( |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 829 | '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php_extension_mode, |
| 830 | rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
ZhouyihaiDing | c8e145b | 2017-09-25 23:20:30 +0000 | [diff] [blame] | 831 | server_language='c++', async_server_threads=1) |
| 832 | |
| 833 | def __str__(self): |
| 834 | return 'php' |
| 835 | |
| 836 | |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 837 | class JavaLanguage: |
| 838 | |
| 839 | def __init__(self): |
| 840 | pass |
| 841 | self.safename = str(self) |
| 842 | |
| 843 | def worker_cmdline(self): |
| 844 | return ['tools/run_tests/performance/run_worker_java.sh'] |
| 845 | |
| 846 | def worker_port_offset(self): |
| 847 | return 400 |
| 848 | |
| 849 | def scenarios(self): |
Jan Tattermusch | e222c00 | 2016-04-20 18:55:24 -0700 | [diff] [blame] | 850 | for secure in [True, False]: |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 851 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 852 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Jan Tattermusch | e222c00 | 2016-04-20 18:55:24 -0700 | [diff] [blame] | 853 | |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 854 | yield _ping_pong_scenario( |
| 855 | 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 856 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
| 857 | use_generic_payload=True, async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 858 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 859 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 860 | |
| 861 | yield _ping_pong_scenario( |
| 862 | 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 863 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 864 | async_server_threads=1, |
| 865 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 866 | |
| 867 | yield _ping_pong_scenario( |
| 868 | 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 869 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 870 | async_server_threads=1, |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 871 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 872 | categories=smoketest_categories) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 873 | |
| 874 | yield _ping_pong_scenario( |
| 875 | 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 876 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 877 | async_server_threads=1, |
| 878 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 879 | |
| 880 | yield _ping_pong_scenario( |
| 881 | 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', |
| 882 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 883 | unconstrained_client='async', |
Jan Tattermusch | 427699b | 2016-05-05 18:10:14 -0700 | [diff] [blame] | 884 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 885 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 886 | |
| 887 | yield _ping_pong_scenario( |
| 888 | 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 889 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 890 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 891 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 892 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 893 | |
| 894 | yield _ping_pong_scenario( |
| 895 | 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 896 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 897 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 898 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, |
| 899 | categories=[SCALABLE]) |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 900 | |
| 901 | yield _ping_pong_scenario( |
| 902 | 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', |
| 903 | client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Vijay Pai | fba000c | 2017-05-23 14:17:17 -0700 | [diff] [blame] | 904 | unconstrained_client='async-limited', use_generic_payload=True, |
Jan Tattermusch | d7b162f | 2016-05-04 15:09:26 -0700 | [diff] [blame] | 905 | async_server_threads=1, |
| 906 | secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) |
| 907 | |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 908 | # TODO(jtattermusch): add scenarios java vs C++ |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 909 | |
| 910 | def __str__(self): |
| 911 | return 'java' |
| 912 | |
| 913 | |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 914 | class GoLanguage: |
| 915 | |
| 916 | def __init__(self): |
| 917 | pass |
| 918 | self.safename = str(self) |
| 919 | |
| 920 | def worker_cmdline(self): |
| 921 | return ['tools/run_tests/performance/run_worker_go.sh'] |
| 922 | |
| 923 | def worker_port_offset(self): |
| 924 | return 600 |
| 925 | |
| 926 | def scenarios(self): |
| 927 | for secure in [True, False]: |
| 928 | secstr = 'secure' if secure else 'insecure' |
Craig Tiller | 677966a | 2016-09-26 07:37:28 -0700 | [diff] [blame] | 929 | smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 930 | |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 931 | # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 932 | # but that's mostly because of lack of better name of the enum value. |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 933 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 934 | 'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
| 935 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 936 | use_generic_payload=True, async_server_threads=1, |
| 937 | secure=secure, |
| 938 | categories=smoketest_categories) |
| 939 | |
| 940 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 941 | 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 942 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 943 | async_server_threads=1, |
| 944 | secure=secure) |
| 945 | |
| 946 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 947 | 'go_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', |
| 948 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
| 949 | async_server_threads=1, |
| 950 | secure=secure, |
| 951 | categories=smoketest_categories) |
| 952 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 953 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 954 | yield _ping_pong_scenario( |
| 955 | 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', |
| 956 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 957 | unconstrained_client='async', |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 958 | secure=secure, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 959 | categories=smoketest_categories+[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 960 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 961 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 962 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 963 | 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 964 | client_type='SYNC_CLIENT', server_type='SYNC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 965 | unconstrained_client='async', |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 966 | secure=secure, |
| 967 | categories=[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 968 | |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 969 | # unconstrained_client='async' is intended (client uses goroutines) |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 970 | # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, |
| 971 | # but that's mostly because of lack of better name of the enum value. |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 972 | yield _ping_pong_scenario( |
Jan Tattermusch | 4bd3a7c | 2016-05-11 09:29:08 -0700 | [diff] [blame] | 973 | 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', |
| 974 | client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', |
Jan Tattermusch | 5cbccd0 | 2016-05-13 16:26:42 -0700 | [diff] [blame] | 975 | unconstrained_client='async', use_generic_payload=True, |
Jan Tattermusch | d27888b | 2016-05-19 16:11:11 -0700 | [diff] [blame] | 976 | secure=secure, |
| 977 | categories=[SCALABLE]) |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 978 | |
Craig Tiller | 4f2d9ae | 2016-07-17 14:07:18 -0700 | [diff] [blame] | 979 | # TODO(jtattermusch): add scenarios go vs C++ |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 980 | |
| 981 | def __str__(self): |
| 982 | return 'go' |
| 983 | |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 984 | class NodeExpressLanguage: |
| 985 | |
| 986 | def __init__(self): |
| 987 | pass |
| 988 | self.safename = str(self) |
| 989 | |
| 990 | def worker_cmdline(self): |
| 991 | return ['tools/run_tests/performance/run_worker_node.sh', |
| 992 | '--benchmark_impl=express'] |
| 993 | |
| 994 | def worker_port_offset(self): |
| 995 | return 700 |
| 996 | |
| 997 | def scenarios(self): |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 998 | yield _ping_pong_scenario( |
murgatroid99 | dabe6bd | 2016-11-09 11:03:21 -0800 | [diff] [blame] | 999 | 'node_express_json_unary_ping_pong', rpc_type='UNARY', |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 1000 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 1001 | categories=[SCALABLE, SMOKETEST]) |
| 1002 | |
| 1003 | yield _ping_pong_scenario( |
murgatroid99 | dabe6bd | 2016-11-09 11:03:21 -0800 | [diff] [blame] | 1004 | 'node_express_json_async_unary_qps_unconstrained', rpc_type='UNARY', |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 1005 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 1006 | unconstrained_client='async', |
| 1007 | categories=[SCALABLE, SMOKETEST]) |
| 1008 | |
murgatroid99 | bf33410 | 2017-04-18 11:11:28 -0700 | [diff] [blame] | 1009 | sizes = [('1B', 1), ('1KB', 1024), ('10KB', 10 * 1024), |
| 1010 | ('1MB', 1024 * 1024), ('10MB', 10 * 1024 * 1024), |
| 1011 | ('100MB', 100 * 1024 * 1024)] |
| 1012 | |
| 1013 | for size_name, size in sizes: |
| 1014 | for secure in (True, False): |
| 1015 | yield _ping_pong_scenario( |
| 1016 | 'node_express_json_unary_ping_pong_%s_resp_%s' % |
| 1017 | (size_name, 'secure' if secure else 'insecure'), |
| 1018 | rpc_type='UNARY', |
| 1019 | client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', |
| 1020 | req_size=0, resp_size=size, |
| 1021 | secure=secure, |
| 1022 | categories=[SCALABLE]) |
| 1023 | |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 1024 | def __str__(self): |
| 1025 | return 'node_express' |
| 1026 | |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 1027 | |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 1028 | LANGUAGES = { |
| 1029 | 'c++' : CXXLanguage(), |
| 1030 | 'csharp' : CSharpLanguage(), |
| 1031 | 'node' : NodeLanguage(), |
murgatroid99 | b53e5d1 | 2016-10-18 09:55:28 -0700 | [diff] [blame] | 1032 | 'node_express': NodeExpressLanguage(), |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 1033 | 'ruby' : RubyLanguage(), |
ZhouyihaiDing | 5fef3e3 | 2017-09-26 18:25:28 +0000 | [diff] [blame] | 1034 | 'php_protobuf_php' : PhpLanguage(), |
| 1035 | 'php_protobuf_c' : PhpLanguage(use_protobuf_c_extension=True), |
Jan Tattermusch | de874a1 | 2016-04-18 09:21:37 -0700 | [diff] [blame] | 1036 | 'java' : JavaLanguage(), |
Ken Payson | 0482c10 | 2016-04-19 12:08:34 -0700 | [diff] [blame] | 1037 | 'python' : PythonLanguage(), |
Jan Tattermusch | 3b59b0f | 2016-05-10 14:44:05 -0700 | [diff] [blame] | 1038 | 'go' : GoLanguage(), |
Craig Tiller | cb2cd26 | 2016-04-01 13:59:54 -0700 | [diff] [blame] | 1039 | } |