Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Sree Kuchibhotla | aecac88 | 2016-04-04 22:48:15 -0700 | [diff] [blame] | 2 | # Copyright 2015-2016, Google Inc. |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 30 | |
| 31 | from __future__ import print_function |
| 32 | |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 33 | import argparse |
| 34 | import datetime |
| 35 | import json |
| 36 | import os |
| 37 | import subprocess |
| 38 | import sys |
| 39 | import time |
| 40 | |
| 41 | stress_test_utils_dir = os.path.abspath(os.path.join( |
| 42 | os.path.dirname(__file__), '../../gcp/stress_test')) |
| 43 | sys.path.append(stress_test_utils_dir) |
| 44 | from stress_test_utils import BigQueryHelper |
| 45 | |
| 46 | kubernetes_api_dir = os.path.abspath(os.path.join( |
| 47 | os.path.dirname(__file__), '../../gcp/utils')) |
| 48 | sys.path.append(kubernetes_api_dir) |
| 49 | |
| 50 | import kubernetes_api |
| 51 | |
| 52 | |
| 53 | class GlobalSettings: |
| 54 | |
| 55 | def __init__(self, gcp_project_id, build_docker_images, |
| 56 | test_poll_interval_secs, test_duration_secs, |
| 57 | kubernetes_proxy_port, dataset_id_prefix, summary_table_id, |
| 58 | qps_table_id, pod_warmup_secs): |
| 59 | self.gcp_project_id = gcp_project_id |
| 60 | self.build_docker_images = build_docker_images |
| 61 | self.test_poll_interval_secs = test_poll_interval_secs |
| 62 | self.test_duration_secs = test_duration_secs |
| 63 | self.kubernetes_proxy_port = kubernetes_proxy_port |
| 64 | self.dataset_id_prefix = dataset_id_prefix |
| 65 | self.summary_table_id = summary_table_id |
| 66 | self.qps_table_id = qps_table_id |
| 67 | self.pod_warmup_secs = pod_warmup_secs |
| 68 | |
| 69 | |
| 70 | class ClientTemplate: |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 71 | """ Contains all the common settings that are used by a stress client """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 72 | |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 73 | def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, |
| 74 | wrapper_script_path, poll_interval_secs, client_args_dict, |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 75 | metrics_args_dict, will_run_forever, env_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 76 | self.name = name |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 77 | self.stress_client_cmd = stress_client_cmd |
| 78 | self.metrics_client_cmd = metrics_client_cmd |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 79 | self.metrics_port = metrics_port |
| 80 | self.wrapper_script_path = wrapper_script_path |
| 81 | self.poll_interval_secs = poll_interval_secs |
| 82 | self.client_args_dict = client_args_dict |
| 83 | self.metrics_args_dict = metrics_args_dict |
Sree Kuchibhotla | 5bc112c | 2016-04-18 15:34:04 -0700 | [diff] [blame] | 84 | self.will_run_forever = will_run_forever |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 85 | self.env_dict = env_dict |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 86 | |
| 87 | |
| 88 | class ServerTemplate: |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 89 | """ Contains all the common settings used by a stress server """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 90 | |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 91 | def __init__(self, name, server_cmd, wrapper_script_path, server_port, |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 92 | server_args_dict, will_run_forever, env_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 93 | self.name = name |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 94 | self.server_cmd = server_cmd |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 95 | self.wrapper_script_path = wrapper_script_path |
| 96 | self.server_port = server_port |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 97 | self.server_args_dict = server_args_dict |
Sree Kuchibhotla | 5bc112c | 2016-04-18 15:34:04 -0700 | [diff] [blame] | 98 | self.will_run_forever = will_run_forever |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 99 | self.env_dict = env_dict |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 100 | |
| 101 | |
| 102 | class DockerImage: |
Sree Kuchibhotla | e68ec43 | 2016-03-28 13:55:01 -0700 | [diff] [blame] | 103 | """ Represents properties of a Docker image. Provides methods to build the |
| 104 | image and push it to GKE registry |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 105 | """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 106 | |
| 107 | def __init__(self, gcp_project_id, image_name, build_script_path, |
Sree Kuchibhotla | 8d41d51 | 2016-03-25 14:50:31 -0700 | [diff] [blame] | 108 | dockerfile_dir, build_type): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 109 | """Args: |
| 110 | |
| 111 | image_name: The docker image name |
| 112 | tag_name: The additional tag name. This is the name used when pushing the |
| 113 | docker image to GKE registry |
| 114 | build_script_path: The path to the build script that builds this docker |
| 115 | image |
| 116 | dockerfile_dir: The name of the directory under |
| 117 | '<grpc_root>/tools/dockerfile' that contains the dockerfile |
| 118 | """ |
| 119 | self.image_name = image_name |
| 120 | self.gcp_project_id = gcp_project_id |
| 121 | self.build_script_path = build_script_path |
| 122 | self.dockerfile_dir = dockerfile_dir |
Sree Kuchibhotla | 8d41d51 | 2016-03-25 14:50:31 -0700 | [diff] [blame] | 123 | self.build_type = build_type |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 124 | self.tag_name = self._make_tag_name(gcp_project_id, image_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 125 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 126 | def _make_tag_name(self, project_id, image_name): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 127 | return 'gcr.io/%s/%s' % (project_id, image_name) |
| 128 | |
| 129 | def build_image(self): |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 130 | print('Building docker image: %s (tag: %s)' % (self.image_name, |
| 131 | self.tag_name)) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 132 | os.environ['INTEROP_IMAGE'] = self.image_name |
| 133 | os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name |
| 134 | os.environ['BASE_NAME'] = self.dockerfile_dir |
Sree Kuchibhotla | 8d41d51 | 2016-03-25 14:50:31 -0700 | [diff] [blame] | 135 | os.environ['BUILD_TYPE'] = self.build_type |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 136 | print('DEBUG: path: ', self.build_script_path) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 137 | if subprocess.call(args=[self.build_script_path]) != 0: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 138 | print('Error in building the Docker image') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 139 | return False |
| 140 | return True |
| 141 | |
| 142 | def push_to_gke_registry(self): |
| 143 | cmd = ['gcloud', 'docker', 'push', self.tag_name] |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 144 | print('Pushing %s to the GKE registry..' % self.tag_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 145 | if subprocess.call(args=cmd) != 0: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 146 | print('Error in pushing the image %s to the GKE registry' % |
| 147 | self.tag_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 148 | return False |
| 149 | return True |
| 150 | |
| 151 | |
| 152 | class ServerPodSpec: |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 153 | """ Contains the information required to launch server pods. """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 154 | |
| 155 | def __init__(self, name, server_template, docker_image, num_instances): |
| 156 | self.name = name |
| 157 | self.template = server_template |
| 158 | self.docker_image = docker_image |
| 159 | self.num_instances = num_instances |
| 160 | |
| 161 | def pod_names(self): |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 162 | """ Return a list of names of server pods to create. """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 163 | return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] |
| 164 | |
| 165 | def server_addresses(self): |
| 166 | """ Return string of server addresses in the following format: |
| 167 | '<server_pod_name_1>:<server_port>,<server_pod_name_2>:<server_port>...' |
| 168 | """ |
| 169 | return ','.join(['%s:%d' % (pod_name, self.template.server_port) |
| 170 | for pod_name in self.pod_names()]) |
| 171 | |
| 172 | |
| 173 | class ClientPodSpec: |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 174 | """ Contains the information required to launch client pods """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 175 | |
| 176 | def __init__(self, name, client_template, docker_image, num_instances, |
| 177 | server_addresses): |
| 178 | self.name = name |
| 179 | self.template = client_template |
| 180 | self.docker_image = docker_image |
| 181 | self.num_instances = num_instances |
| 182 | self.server_addresses = server_addresses |
| 183 | |
| 184 | def pod_names(self): |
| 185 | """ Return a list of names of client pods to create """ |
| 186 | return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] |
| 187 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 188 | # The client args in the template do not have server addresses. This function |
| 189 | # adds the server addresses and returns the updated client args |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 190 | def get_client_args_dict(self): |
| 191 | args_dict = self.template.client_args_dict.copy() |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 192 | args_dict['server_addresses'] = self.server_addresses |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 193 | return args_dict |
| 194 | |
| 195 | |
| 196 | class Gke: |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 197 | """ Class that has helper methods to interact with GKE """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 198 | |
| 199 | class KubernetesProxy: |
| 200 | """Class to start a proxy on localhost to talk to the Kubernetes API server""" |
| 201 | |
| 202 | def __init__(self, port): |
| 203 | cmd = ['kubectl', 'proxy', '--port=%d' % port] |
| 204 | self.p = subprocess.Popen(args=cmd) |
| 205 | time.sleep(2) |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 206 | print('\nStarted kubernetes proxy on port: %d' % port) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 207 | |
| 208 | def __del__(self): |
| 209 | if self.p is not None: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 210 | print('Shutting down Kubernetes proxy..') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 211 | self.p.kill() |
| 212 | |
| 213 | def __init__(self, project_id, run_id, dataset_id, summary_table_id, |
| 214 | qps_table_id, kubernetes_port): |
| 215 | self.project_id = project_id |
| 216 | self.run_id = run_id |
| 217 | self.dataset_id = dataset_id |
| 218 | self.summary_table_id = summary_table_id |
| 219 | self.qps_table_id = qps_table_id |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 220 | |
| 221 | # The environment variables we would like to pass to every pod (both client |
| 222 | # and server) launched in GKE |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 223 | self.gke_env = { |
| 224 | 'RUN_ID': self.run_id, |
| 225 | 'GCP_PROJECT_ID': self.project_id, |
| 226 | 'DATASET_ID': self.dataset_id, |
| 227 | 'SUMMARY_TABLE_ID': self.summary_table_id, |
| 228 | 'QPS_TABLE_ID': self.qps_table_id |
| 229 | } |
| 230 | |
| 231 | self.kubernetes_port = kubernetes_port |
| 232 | # Start kubernetes proxy |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 233 | self.kubernetes_proxy = Gke.KubernetesProxy(kubernetes_port) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 234 | |
| 235 | def _args_dict_to_str(self, args_dict): |
| 236 | return ' '.join('--%s=%s' % (k, args_dict[k]) for k in args_dict.keys()) |
| 237 | |
| 238 | def launch_servers(self, server_pod_spec): |
| 239 | is_success = True |
| 240 | |
| 241 | # The command to run inside the container is the wrapper script (which then |
| 242 | # launches the actual server) |
| 243 | container_cmd = server_pod_spec.template.wrapper_script_path |
| 244 | |
| 245 | # The parameters to the wrapper script (defined in |
| 246 | # server_pod_spec.template.wrapper_script_path) are are injected into the |
| 247 | # container via environment variables |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 248 | server_env = self.gke_env.copy() |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 249 | server_env.update(server_pod_spec.template.env_dict) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 250 | server_env.update({ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 251 | 'STRESS_TEST_IMAGE_TYPE': 'SERVER', |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 252 | 'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 253 | 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( |
Sree Kuchibhotla | 5bc112c | 2016-04-18 15:34:04 -0700 | [diff] [blame] | 254 | server_pod_spec.template.server_args_dict), |
| 255 | 'WILL_RUN_FOREVER': str(server_pod_spec.template.will_run_forever) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 256 | }) |
| 257 | |
| 258 | for pod_name in server_pod_spec.pod_names(): |
| 259 | server_env['POD_NAME'] = pod_name |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 260 | print('Creating server: %s' % pod_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 261 | is_success = kubernetes_api.create_pod_and_service( |
| 262 | 'localhost', |
| 263 | self.kubernetes_port, |
| 264 | 'default', # Use 'default' namespace |
| 265 | pod_name, |
| 266 | server_pod_spec.docker_image.tag_name, |
| 267 | [server_pod_spec.template.server_port], # Ports to expose on the pod |
| 268 | [container_cmd], |
| 269 | [], # Args list is empty since we are passing all args via env variables |
| 270 | server_env, |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 271 | True # Headless = True for server to that GKE creates a DNS record for pod_name |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 272 | ) |
| 273 | if not is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 274 | print('Error in launching server: %s' % pod_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 275 | break |
| 276 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 277 | if is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 278 | print('Successfully created server(s)') |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 279 | |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 280 | return is_success |
| 281 | |
| 282 | def launch_clients(self, client_pod_spec): |
| 283 | is_success = True |
| 284 | |
| 285 | # The command to run inside the container is the wrapper script (which then |
| 286 | # launches the actual stress client) |
| 287 | container_cmd = client_pod_spec.template.wrapper_script_path |
| 288 | |
| 289 | # The parameters to the wrapper script (defined in |
| 290 | # client_pod_spec.template.wrapper_script_path) are are injected into the |
| 291 | # container via environment variables |
| 292 | client_env = self.gke_env.copy() |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 293 | client_env.update(client_pod_spec.template.env_dict) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 294 | client_env.update({ |
| 295 | 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 296 | 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 297 | 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( |
| 298 | client_pod_spec.get_client_args_dict()), |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 299 | 'METRICS_CLIENT_CMD': client_pod_spec.template.metrics_client_cmd, |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 300 | 'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str( |
| 301 | client_pod_spec.template.metrics_args_dict), |
Sree Kuchibhotla | 5bc112c | 2016-04-18 15:34:04 -0700 | [diff] [blame] | 302 | 'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs), |
| 303 | 'WILL_RUN_FOREVER': str(client_pod_spec.template.will_run_forever) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 304 | }) |
| 305 | |
| 306 | for pod_name in client_pod_spec.pod_names(): |
| 307 | client_env['POD_NAME'] = pod_name |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 308 | print('Creating client: %s' % pod_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 309 | is_success = kubernetes_api.create_pod_and_service( |
| 310 | 'localhost', |
| 311 | self.kubernetes_port, |
| 312 | 'default', # default namespace, |
| 313 | pod_name, |
| 314 | client_pod_spec.docker_image.tag_name, |
| 315 | [client_pod_spec.template.metrics_port], # Ports to expose on the pod |
| 316 | [container_cmd], |
| 317 | [], # Empty args list since all args are passed via env variables |
| 318 | client_env, |
Sree Kuchibhotla | 081b603 | 2017-02-02 16:52:30 -0800 | [diff] [blame] | 319 | True # Client is a headless service (no need for an external ip) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 320 | ) |
| 321 | |
| 322 | if not is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 323 | print('Error in launching client %s' % pod_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 324 | break |
| 325 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 326 | if is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 327 | print('Successfully created all client(s)') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 328 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 329 | return is_success |
| 330 | |
| 331 | def _delete_pods(self, pod_name_list): |
| 332 | is_success = True |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 333 | for pod_name in pod_name_list: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 334 | print('Deleting %s' % pod_name) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 335 | is_success = kubernetes_api.delete_pod_and_service( |
| 336 | 'localhost', |
| 337 | self.kubernetes_port, |
| 338 | 'default', # default namespace |
| 339 | pod_name) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 340 | |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 341 | if not is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 342 | print('Error in deleting pod %s' % pod_name) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 343 | break |
| 344 | |
| 345 | if is_success: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 346 | print('Successfully deleted all pods') |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 347 | |
| 348 | return is_success |
| 349 | |
| 350 | def delete_servers(self, server_pod_spec): |
| 351 | return self._delete_pods(server_pod_spec.pod_names()) |
| 352 | |
| 353 | def delete_clients(self, client_pod_spec): |
| 354 | return self._delete_pods(client_pod_spec.pod_names()) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 355 | |
| 356 | |
| 357 | class Config: |
| 358 | |
Sree Kuchibhotla | 815c589 | 2016-03-25 15:03:50 -0700 | [diff] [blame] | 359 | def __init__(self, config_filename, gcp_project_id): |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 360 | print('Loading configuration...') |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 361 | config_dict = self._load_config(config_filename) |
| 362 | |
| 363 | self.global_settings = self._parse_global_settings(config_dict, |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 364 | gcp_project_id) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 365 | self.docker_images_dict = self._parse_docker_images( |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 366 | config_dict, self.global_settings.gcp_project_id) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 367 | self.client_templates_dict = self._parse_client_templates(config_dict) |
| 368 | self.server_templates_dict = self._parse_server_templates(config_dict) |
| 369 | self.server_pod_specs_dict = self._parse_server_pod_specs( |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 370 | config_dict, self.docker_images_dict, self.server_templates_dict) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 371 | self.client_pod_specs_dict = self._parse_client_pod_specs( |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 372 | config_dict, self.docker_images_dict, self.client_templates_dict, |
| 373 | self.server_pod_specs_dict) |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 374 | print('Loaded Configuaration.') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 375 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 376 | def _parse_global_settings(self, config_dict, gcp_project_id): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 377 | global_settings_dict = config_dict['globalSettings'] |
Sree Kuchibhotla | 815c589 | 2016-03-25 15:03:50 -0700 | [diff] [blame] | 378 | return GlobalSettings(gcp_project_id, |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 379 | global_settings_dict['buildDockerImages'], |
| 380 | global_settings_dict['pollIntervalSecs'], |
| 381 | global_settings_dict['testDurationSecs'], |
| 382 | global_settings_dict['kubernetesProxyPort'], |
| 383 | global_settings_dict['datasetIdNamePrefix'], |
| 384 | global_settings_dict['summaryTableId'], |
| 385 | global_settings_dict['qpsTableId'], |
| 386 | global_settings_dict['podWarmupSecs']) |
| 387 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 388 | def _parse_docker_images(self, config_dict, gcp_project_id): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 389 | """Parses the 'dockerImages' section of the config file and returns a |
| 390 | Dictionary of 'DockerImage' objects keyed by docker image names""" |
| 391 | docker_images_dict = {} |
Sree Kuchibhotla | 8d41d51 | 2016-03-25 14:50:31 -0700 | [diff] [blame] | 392 | |
| 393 | docker_config_dict = config_dict['dockerImages'] |
| 394 | for image_name in docker_config_dict.keys(): |
| 395 | build_script_path = docker_config_dict[image_name]['buildScript'] |
| 396 | dockerfile_dir = docker_config_dict[image_name]['dockerFileDir'] |
Sree Kuchibhotla | 7a05436 | 2016-04-04 14:08:02 -0700 | [diff] [blame] | 397 | build_type = docker_config_dict[image_name].get('buildType', 'opt') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 398 | docker_images_dict[image_name] = DockerImage(gcp_project_id, image_name, |
| 399 | build_script_path, |
Sree Kuchibhotla | 8d41d51 | 2016-03-25 14:50:31 -0700 | [diff] [blame] | 400 | dockerfile_dir, build_type) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 401 | return docker_images_dict |
| 402 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 403 | def _parse_client_templates(self, config_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 404 | """Parses the 'clientTemplates' section of the config file and returns a |
| 405 | Dictionary of 'ClientTemplate' objects keyed by client template names. |
| 406 | |
| 407 | Note: The 'baseTemplates' sub section of the config file contains templates |
| 408 | with default values and the 'templates' sub section contains the actual |
| 409 | client templates (which refer to the base template name to use for default |
| 410 | values). |
| 411 | """ |
| 412 | client_templates_dict = {} |
| 413 | |
| 414 | templates_dict = config_dict['clientTemplates']['templates'] |
| 415 | base_templates_dict = config_dict['clientTemplates'].get('baseTemplates', |
| 416 | {}) |
| 417 | for template_name in templates_dict.keys(): |
| 418 | # temp_dict is a temporary dictionary that merges base template dictionary |
| 419 | # and client template dictionary (with client template dictionary values |
| 420 | # overriding base template values) |
| 421 | temp_dict = {} |
| 422 | |
| 423 | base_template_name = templates_dict[template_name].get('baseTemplate') |
| 424 | if not base_template_name is None: |
| 425 | temp_dict = base_templates_dict[base_template_name].copy() |
| 426 | |
| 427 | temp_dict.update(templates_dict[template_name]) |
| 428 | |
| 429 | # Create and add ClientTemplate object to the final client_templates_dict |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 430 | stress_client_cmd = ' '.join(temp_dict['stressClientCmd']) |
| 431 | metrics_client_cmd = ' '.join(temp_dict['metricsClientCmd']) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 432 | client_templates_dict[template_name] = ClientTemplate( |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 433 | template_name, stress_client_cmd, metrics_client_cmd, |
| 434 | temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], |
| 435 | temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 436 | temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1), |
| 437 | temp_dict.get('env', {}).copy()) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 438 | |
| 439 | return client_templates_dict |
| 440 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 441 | def _parse_server_templates(self, config_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 442 | """Parses the 'serverTemplates' section of the config file and returns a |
| 443 | Dictionary of 'serverTemplate' objects keyed by server template names. |
| 444 | |
| 445 | Note: The 'baseTemplates' sub section of the config file contains templates |
| 446 | with default values and the 'templates' sub section contains the actual |
| 447 | server templates (which refer to the base template name to use for default |
| 448 | values). |
| 449 | """ |
| 450 | server_templates_dict = {} |
| 451 | |
| 452 | templates_dict = config_dict['serverTemplates']['templates'] |
| 453 | base_templates_dict = config_dict['serverTemplates'].get('baseTemplates', |
| 454 | {}) |
| 455 | |
| 456 | for template_name in templates_dict.keys(): |
| 457 | # temp_dict is a temporary dictionary that merges base template dictionary |
| 458 | # and server template dictionary (with server template dictionary values |
| 459 | # overriding base template values) |
| 460 | temp_dict = {} |
| 461 | |
| 462 | base_template_name = templates_dict[template_name].get('baseTemplate') |
| 463 | if not base_template_name is None: |
| 464 | temp_dict = base_templates_dict[base_template_name].copy() |
| 465 | |
| 466 | temp_dict.update(templates_dict[template_name]) |
| 467 | |
| 468 | # Create and add ServerTemplate object to the final server_templates_dict |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 469 | stress_server_cmd = ' '.join(temp_dict['stressServerCmd']) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 470 | server_templates_dict[template_name] = ServerTemplate( |
Sree Kuchibhotla | 83a9a65 | 2016-04-04 21:49:15 -0700 | [diff] [blame] | 471 | template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], |
Sree Kuchibhotla | 5bc112c | 2016-04-18 15:34:04 -0700 | [diff] [blame] | 472 | temp_dict['serverPort'], temp_dict['serverArgs'].copy(), |
Sree Kuchibhotla | 1bdd531 | 2016-05-02 19:57:04 -0700 | [diff] [blame] | 473 | temp_dict.get('willRunForever', 1), temp_dict.get('env', {}).copy()) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 474 | |
| 475 | return server_templates_dict |
| 476 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 477 | def _parse_server_pod_specs(self, config_dict, docker_images_dict, |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 478 | server_templates_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 479 | """Parses the 'serverPodSpecs' sub-section (under 'testMatrix' section) of |
| 480 | the config file and returns a Dictionary of 'ServerPodSpec' objects keyed |
| 481 | by server pod spec names""" |
| 482 | server_pod_specs_dict = {} |
| 483 | |
| 484 | pod_specs_dict = config_dict['testMatrix'].get('serverPodSpecs', {}) |
| 485 | |
| 486 | for pod_name in pod_specs_dict.keys(): |
| 487 | server_template_name = pod_specs_dict[pod_name]['serverTemplate'] |
| 488 | docker_image_name = pod_specs_dict[pod_name]['dockerImage'] |
| 489 | num_instances = pod_specs_dict[pod_name].get('numInstances', 1) |
| 490 | |
| 491 | # Create and add the ServerPodSpec object to the final |
| 492 | # server_pod_specs_dict |
| 493 | server_pod_specs_dict[pod_name] = ServerPodSpec( |
| 494 | pod_name, server_templates_dict[server_template_name], |
| 495 | docker_images_dict[docker_image_name], num_instances) |
| 496 | |
| 497 | return server_pod_specs_dict |
| 498 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 499 | def _parse_client_pod_specs(self, config_dict, docker_images_dict, |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 500 | client_templates_dict, server_pod_specs_dict): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 501 | """Parses the 'clientPodSpecs' sub-section (under 'testMatrix' section) of |
| 502 | the config file and returns a Dictionary of 'ClientPodSpec' objects keyed |
| 503 | by client pod spec names""" |
| 504 | client_pod_specs_dict = {} |
| 505 | |
| 506 | pod_specs_dict = config_dict['testMatrix'].get('clientPodSpecs', {}) |
| 507 | for pod_name in pod_specs_dict.keys(): |
| 508 | client_template_name = pod_specs_dict[pod_name]['clientTemplate'] |
| 509 | docker_image_name = pod_specs_dict[pod_name]['dockerImage'] |
| 510 | num_instances = pod_specs_dict[pod_name]['numInstances'] |
| 511 | |
| 512 | # Get the server addresses from the server pod spec object |
| 513 | server_pod_spec_name = pod_specs_dict[pod_name]['serverPodSpec'] |
| 514 | server_addresses = server_pod_specs_dict[ |
| 515 | server_pod_spec_name].server_addresses() |
| 516 | |
| 517 | client_pod_specs_dict[pod_name] = ClientPodSpec( |
| 518 | pod_name, client_templates_dict[client_template_name], |
| 519 | docker_images_dict[docker_image_name], num_instances, |
| 520 | server_addresses) |
| 521 | |
| 522 | return client_pod_specs_dict |
| 523 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 524 | def _load_config(self, config_filename): |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 525 | """Opens the config file and converts the Json text to Dictionary""" |
| 526 | if not os.path.isabs(config_filename): |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 527 | raise Exception('Config objects expects an absolute file path. ' |
| 528 | 'config file name passed: %s' % config_filename) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 529 | with open(config_filename) as config_file: |
| 530 | return json.load(config_file) |
| 531 | |
| 532 | |
| 533 | def run_tests(config): |
Sree Kuchibhotla | 9c9644b | 2016-03-28 09:30:51 -0700 | [diff] [blame] | 534 | """ The main function that launches the stress tests """ |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 535 | # Build docker images and push to GKE registry |
| 536 | if config.global_settings.build_docker_images: |
| 537 | for name, docker_image in config.docker_images_dict.iteritems(): |
| 538 | if not (docker_image.build_image() and |
| 539 | docker_image.push_to_gke_registry()): |
| 540 | return False |
| 541 | |
| 542 | # Create a unique id for this run (Note: Using timestamp instead of UUID to |
| 543 | # make it easier to deduce the date/time of the run just by looking at the run |
| 544 | # run id. This is useful in debugging when looking at records in Biq query) |
| 545 | run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') |
| 546 | dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 547 | print('Run id:', run_id) |
| 548 | print('Dataset id:', dataset_id) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 549 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 550 | bq_helper = BigQueryHelper(run_id, '', '', |
| 551 | config.global_settings.gcp_project_id, dataset_id, |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 552 | config.global_settings.summary_table_id, |
| 553 | config.global_settings.qps_table_id) |
| 554 | bq_helper.initialize() |
| 555 | |
| 556 | gke = Gke(config.global_settings.gcp_project_id, run_id, dataset_id, |
| 557 | config.global_settings.summary_table_id, |
| 558 | config.global_settings.qps_table_id, |
| 559 | config.global_settings.kubernetes_proxy_port) |
| 560 | |
| 561 | is_success = True |
| 562 | |
| 563 | try: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 564 | print('Launching servers..') |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 565 | for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
| 566 | if not gke.launch_servers(server_pod_spec): |
| 567 | is_success = False # is_success is checked in the 'finally' block |
| 568 | return False |
| 569 | |
| 570 | print('Launched servers. Waiting for %d seconds for the server pods to be ' |
| 571 | 'fully online') % config.global_settings.pod_warmup_secs |
| 572 | time.sleep(config.global_settings.pod_warmup_secs) |
| 573 | |
| 574 | for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
| 575 | if not gke.launch_clients(client_pod_spec): |
| 576 | is_success = False # is_success is checked in the 'finally' block |
| 577 | return False |
| 578 | |
| 579 | print('Launched all clients. Waiting for %d seconds for the client pods to ' |
| 580 | 'be fully online') % config.global_settings.pod_warmup_secs |
| 581 | time.sleep(config.global_settings.pod_warmup_secs) |
| 582 | |
| 583 | start_time = datetime.datetime.now() |
| 584 | end_time = start_time + datetime.timedelta( |
| 585 | seconds=config.global_settings.test_duration_secs) |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 586 | print('Running the test until %s' % end_time.isoformat()) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 587 | |
| 588 | while True: |
| 589 | if datetime.datetime.now() > end_time: |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 590 | print('Test was run for %d seconds' % |
| 591 | config.global_settings.test_duration_secs) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 592 | break |
| 593 | |
| 594 | # Check if either stress server or clients have failed (btw, the bq_helper |
| 595 | # monitors all the rows in the summary table and checks if any of them |
| 596 | # have a failure status) |
| 597 | if bq_helper.check_if_any_tests_failed(): |
| 598 | is_success = False |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 599 | print('Some tests failed.') |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 600 | break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 601 | |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 602 | # Tests running fine. Wait until next poll time to check the status |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 603 | print('Sleeping for %d seconds..' % |
| 604 | config.global_settings.test_poll_interval_secs) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 605 | time.sleep(config.global_settings.test_poll_interval_secs) |
| 606 | |
| 607 | # Print BiqQuery tables |
| 608 | bq_helper.print_qps_records() |
| 609 | bq_helper.print_summary_records() |
| 610 | |
| 611 | finally: |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 612 | # If there was a test failure, we should not delete the pods since they |
| 613 | # would contain useful debug information (logs, core dumps etc) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 614 | if is_success: |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 615 | for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
| 616 | gke.delete_servers(server_pod_spec) |
| 617 | for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
| 618 | gke.delete_clients(client_pod_spec) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 619 | |
| 620 | return is_success |
| 621 | |
| 622 | |
Sree Kuchibhotla | 478bd44 | 2016-04-12 17:40:24 -0700 | [diff] [blame] | 623 | def tear_down(config): |
| 624 | gke = Gke(config.global_settings.gcp_project_id, '', '', |
| 625 | config.global_settings.summary_table_id, |
| 626 | config.global_settings.qps_table_id, |
| 627 | config.global_settings.kubernetes_proxy_port) |
| 628 | for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
| 629 | gke.delete_servers(server_pod_spec) |
| 630 | for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
| 631 | gke.delete_clients(client_pod_spec) |
| 632 | |
| 633 | |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 634 | argp = argparse.ArgumentParser( |
| 635 | description='Launch stress tests in GKE', |
| 636 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
Sree Kuchibhotla | 815c589 | 2016-03-25 15:03:50 -0700 | [diff] [blame] | 637 | argp.add_argument('--gcp_project_id', |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 638 | required=True, |
| 639 | help='The Google Cloud Platform Project Id') |
| 640 | argp.add_argument('--config_file', |
| 641 | required=True, |
| 642 | type=str, |
| 643 | help='The test config file') |
Sree Kuchibhotla | 478bd44 | 2016-04-12 17:40:24 -0700 | [diff] [blame] | 644 | argp.add_argument('--tear_down', action='store_true', default=False) |
Sree Kuchibhotla | 575f0fa | 2016-03-25 14:27:07 -0700 | [diff] [blame] | 645 | |
| 646 | if __name__ == '__main__': |
| 647 | args = argp.parse_args() |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 648 | |
| 649 | config_filename = args.config_file |
| 650 | |
Sree Kuchibhotla | 5cadf51 | 2016-03-28 08:55:11 -0700 | [diff] [blame] | 651 | # Since we will be changing the current working directory to grpc root in the |
| 652 | # next step, we should check if the config filename path is a relative path |
| 653 | # (i.e a path relative to the current working directory) and if so, convert it |
| 654 | # to abosulte path |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 655 | if not os.path.isabs(config_filename): |
Sree Kuchibhotla | 5cadf51 | 2016-03-28 08:55:11 -0700 | [diff] [blame] | 656 | config_filename = os.path.abspath(config_filename) |
Sree Kuchibhotla | cdf7734 | 2016-03-25 15:37:34 -0700 | [diff] [blame] | 657 | |
| 658 | config = Config(config_filename, args.gcp_project_id) |
| 659 | |
| 660 | # Change current working directory to grpc root |
| 661 | # (This is important because all relative file paths in the config file are |
| 662 | # supposed to interpreted as relative to the GRPC root) |
| 663 | grpc_root = os.path.abspath(os.path.join( |
| 664 | os.path.dirname(sys.argv[0]), '../../..')) |
| 665 | os.chdir(grpc_root) |
| 666 | |
Sree Kuchibhotla | 478bd44 | 2016-04-12 17:40:24 -0700 | [diff] [blame] | 667 | # Note that tear_down is only in cases where we want to manually tear down a |
| 668 | # test that for some reason run_tests() could not cleanup |
| 669 | if args.tear_down: |
| 670 | tear_down(config) |
| 671 | sys.exit(1) |
| 672 | |
Sree Kuchibhotla | 247b34f | 2016-03-29 16:56:14 -0700 | [diff] [blame] | 673 | if not run_tests(config): |
| 674 | sys.exit(1) |