Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 1 | # Copyright 2015 gRPC authors. |
Craig Tiller | c2c7921 | 2015-02-16 12:00:01 -0800 | [diff] [blame] | 2 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
Craig Tiller | c2c7921 | 2015-02-16 12:00:01 -0800 | [diff] [blame] | 6 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
Craig Tiller | c2c7921 | 2015-02-16 12:00:01 -0800 | [diff] [blame] | 8 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 14 | """Run a group of subprocesses and then finish.""" |
| 15 | |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 16 | from __future__ import print_function |
| 17 | |
David Garcia Quintas | c30b84e | 2017-02-21 15:20:09 -0800 | [diff] [blame] | 18 | import logging |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 19 | import multiprocessing |
Craig Tiller | 7173518 | 2015-01-15 17:07:13 -0800 | [diff] [blame] | 20 | import os |
Craig Tiller | 5058c69 | 2015-04-08 09:42:04 -0700 | [diff] [blame] | 21 | import platform |
Craig Tiller | 5f735a6 | 2016-01-20 09:31:15 -0800 | [diff] [blame] | 22 | import re |
Craig Tiller | 336ad50 | 2015-02-24 14:46:02 -0800 | [diff] [blame] | 23 | import signal |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 24 | import subprocess |
| 25 | import sys |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 26 | import tempfile |
| 27 | import time |
Craig Tiller | 46356b7 | 2017-05-26 15:22:08 +0000 | [diff] [blame] | 28 | import errno |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 29 | |
Craig Tiller | 5f735a6 | 2016-01-20 09:31:15 -0800 | [diff] [blame] | 30 | # cpu cost measurement |
| 31 | measure_cpu_costs = False |
| 32 | |
ctiller | 94e5dde | 2015-01-09 10:41:59 -0800 | [diff] [blame] | 33 | _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 34 | _MAX_RESULT_SIZE = 8192 |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 35 | |
Mark D. Roth | 6288542 | 2016-12-02 15:44:04 +0000 | [diff] [blame] | 36 | |
Mark D. Roth | 158a4a4 | 2016-12-02 10:53:26 -0800 | [diff] [blame] | 37 | # NOTE: If you change this, please make sure to test reviewing the |
| 38 | # github PR with http://reviewable.io, which is known to add UTF-8 |
| 39 | # characters to the PR description, which leak into the environment here |
| 40 | # and cause failures. |
Mark D. Roth | 6288542 | 2016-12-02 15:44:04 +0000 | [diff] [blame] | 41 | def strip_non_ascii_chars(s): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 42 | return ''.join(c for c in s if ord(c) < 128) |
Mark D. Roth | 6288542 | 2016-12-02 15:44:04 +0000 | [diff] [blame] | 43 | |
| 44 | |
Masood Malekghassemi | 768b1db | 2016-06-06 16:45:19 -0700 | [diff] [blame] | 45 | def sanitized_environment(env): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 46 | sanitized = {} |
| 47 | for key, value in env.items(): |
| 48 | sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) |
| 49 | return sanitized |
Masood Malekghassemi | 768b1db | 2016-06-06 16:45:19 -0700 | [diff] [blame] | 50 | |
Mark D. Roth | 6288542 | 2016-12-02 15:44:04 +0000 | [diff] [blame] | 51 | |
Nicolas "Pixel" Noble | f72d7b5 | 2015-12-03 03:07:43 +0100 | [diff] [blame] | 52 | def platform_string(): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 53 | if platform.system() == 'Windows': |
| 54 | return 'windows' |
| 55 | elif platform.system()[:7] == 'MSYS_NT': |
| 56 | return 'windows' |
| 57 | elif platform.system() == 'Darwin': |
| 58 | return 'mac' |
| 59 | elif platform.system() == 'Linux': |
| 60 | return 'linux' |
| 61 | else: |
| 62 | return 'posix' |
Nicolas "Pixel" Noble | f72d7b5 | 2015-12-03 03:07:43 +0100 | [diff] [blame] | 63 | |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 64 | |
Craig Tiller | 336ad50 | 2015-02-24 14:46:02 -0800 | [diff] [blame] | 65 | # setup a signal handler so that signal.pause registers 'something' |
| 66 | # when a child finishes |
| 67 | # not using futures and threading to avoid a dependency on subprocess32 |
Nicolas "Pixel" Noble | f72d7b5 | 2015-12-03 03:07:43 +0100 | [diff] [blame] | 68 | if platform_string() == 'windows': |
Matt Kwong | 69ce380 | 2017-09-12 13:30:51 -0700 | [diff] [blame] | 69 | pass |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 70 | else: |
Craig Tiller | 5058c69 | 2015-04-08 09:42:04 -0700 | [diff] [blame] | 71 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 72 | def alarm_handler(unused_signum, unused_frame): |
| 73 | pass |
Craig Tiller | 336ad50 | 2015-02-24 14:46:02 -0800 | [diff] [blame] | 74 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 75 | signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) |
| 76 | signal.signal(signal.SIGALRM, alarm_handler) |
Craig Tiller | 336ad50 | 2015-02-24 14:46:02 -0800 | [diff] [blame] | 77 | |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 78 | _SUCCESS = object() |
| 79 | _FAILURE = object() |
| 80 | _RUNNING = object() |
| 81 | _KILLED = object() |
| 82 | |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 83 | _COLORS = { |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 84 | 'red': [31, 0], |
| 85 | 'green': [32, 0], |
| 86 | 'yellow': [33, 0], |
| 87 | 'lightgray': [37, 0], |
| 88 | 'gray': [30, 1], |
| 89 | 'purple': [35, 0], |
| 90 | 'cyan': [36, 0] |
| 91 | } |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 92 | |
| 93 | _BEGINNING_OF_LINE = '\x1b[0G' |
| 94 | _CLEAR_LINE = '\x1b[2K' |
| 95 | |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 96 | _TAG_COLOR = { |
| 97 | 'FAILED': 'red', |
Craig Tiller | d7e09c3 | 2015-09-25 11:33:39 -0700 | [diff] [blame] | 98 | 'FLAKE': 'purple', |
Craig Tiller | 3dc1e4f | 2015-09-25 11:46:56 -0700 | [diff] [blame] | 99 | 'TIMEOUT_FLAKE': 'purple', |
Masood Malekghassemi | e5f7002 | 2015-06-29 09:20:26 -0700 | [diff] [blame] | 100 | 'WARNING': 'yellow', |
Craig Tiller | e1d0d1c | 2015-02-27 08:54:23 -0800 | [diff] [blame] | 101 | 'TIMEOUT': 'red', |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 102 | 'PASSED': 'green', |
Nicolas Noble | 044db74 | 2015-01-14 16:57:24 -0800 | [diff] [blame] | 103 | 'START': 'gray', |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 104 | 'WAITING': 'yellow', |
Nicolas Noble | 044db74 | 2015-01-14 16:57:24 -0800 | [diff] [blame] | 105 | 'SUCCESS': 'green', |
| 106 | 'IDLE': 'gray', |
Matt Kwong | 5c691c6 | 2016-10-20 17:11:18 -0700 | [diff] [blame] | 107 | 'SKIPPED': 'cyan' |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 108 | } |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 109 | |
David Garcia Quintas | e6e7b01 | 2017-02-22 17:13:53 -0800 | [diff] [blame] | 110 | _FORMAT = '%(asctime)-15s %(message)s' |
| 111 | logging.basicConfig(level=logging.INFO, format=_FORMAT) |
| 112 | |
Craig Tiller | 46356b7 | 2017-05-26 15:22:08 +0000 | [diff] [blame] | 113 | |
| 114 | def eintr_be_gone(fn): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 115 | """Run fn until it doesn't stop because of EINTR""" |
| 116 | while True: |
| 117 | try: |
| 118 | return fn() |
| 119 | except IOError, e: |
| 120 | if e.errno != errno.EINTR: |
| 121 | raise |
Craig Tiller | 46356b7 | 2017-05-26 15:22:08 +0000 | [diff] [blame] | 122 | |
| 123 | |
Nicolas "Pixel" Noble | 99768ac | 2015-05-13 02:34:06 +0200 | [diff] [blame] | 124 | def message(tag, msg, explanatory_text=None, do_newline=False): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 125 | if message.old_tag == tag and message.old_msg == msg and not explanatory_text: |
| 126 | return |
| 127 | message.old_tag = tag |
| 128 | message.old_msg = msg |
| 129 | while True: |
| 130 | try: |
| 131 | if platform_string() == 'windows' or not sys.stdout.isatty(): |
| 132 | if explanatory_text: |
| 133 | logging.info(explanatory_text) |
| 134 | logging.info('%s: %s', tag, msg) |
| 135 | else: |
| 136 | sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % ( |
| 137 | _BEGINNING_OF_LINE, _CLEAR_LINE, '\n%s' % explanatory_text |
| 138 | if explanatory_text is not None else '', |
| 139 | _COLORS[_TAG_COLOR[tag]][1], _COLORS[_TAG_COLOR[tag]][0], |
| 140 | tag, msg, '\n' |
| 141 | if do_newline or explanatory_text is not None else '')) |
| 142 | sys.stdout.flush() |
| 143 | return |
| 144 | except IOError, e: |
| 145 | if e.errno != errno.EINTR: |
| 146 | raise |
| 147 | |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 148 | |
Adele Zhou | e4c3561 | 2015-10-16 15:34:23 -0700 | [diff] [blame] | 149 | message.old_tag = '' |
| 150 | message.old_msg = '' |
Craig Tiller | 3b08306 | 2015-01-12 13:51:28 -0800 | [diff] [blame] | 151 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 152 | |
Craig Tiller | 7173518 | 2015-01-15 17:07:13 -0800 | [diff] [blame] | 153 | def which(filename): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 154 | if '/' in filename: |
| 155 | return filename |
| 156 | for path in os.environ['PATH'].split(os.pathsep): |
| 157 | if os.path.exists(os.path.join(path, filename)): |
| 158 | return os.path.join(path, filename) |
| 159 | raise Exception('%s not found' % filename) |
Craig Tiller | 7173518 | 2015-01-15 17:07:13 -0800 | [diff] [blame] | 160 | |
| 161 | |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 162 | class JobSpec(object): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 163 | """Specifies what to run for a job.""" |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 164 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 165 | def __init__(self, |
| 166 | cmdline, |
| 167 | shortname=None, |
| 168 | environ=None, |
| 169 | cwd=None, |
| 170 | shell=False, |
| 171 | timeout_seconds=5 * 60, |
| 172 | flake_retries=0, |
| 173 | timeout_retries=0, |
| 174 | kill_handler=None, |
| 175 | cpu_cost=1.0, |
| 176 | verbose_success=False): |
| 177 | """ |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 178 | Arguments: |
| 179 | cmdline: a list of arguments to pass as the command line |
| 180 | environ: a dictionary of environment variables to set in the child process |
Jan Tattermusch | e268628 | 2015-10-08 16:27:07 -0700 | [diff] [blame] | 181 | kill_handler: a handler that will be called whenever job.kill() is invoked |
Craig Tiller | 56c6b6a | 2016-01-20 08:27:37 -0800 | [diff] [blame] | 182 | cpu_cost: number of cores per second this job needs |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 183 | """ |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 184 | if environ is None: |
| 185 | environ = {} |
| 186 | self.cmdline = cmdline |
| 187 | self.environ = environ |
| 188 | self.shortname = cmdline[0] if shortname is None else shortname |
| 189 | self.cwd = cwd |
| 190 | self.shell = shell |
| 191 | self.timeout_seconds = timeout_seconds |
| 192 | self.flake_retries = flake_retries |
| 193 | self.timeout_retries = timeout_retries |
| 194 | self.kill_handler = kill_handler |
| 195 | self.cpu_cost = cpu_cost |
| 196 | self.verbose_success = verbose_success |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 197 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 198 | def identity(self): |
| 199 | return '%r %r' % (self.cmdline, self.environ) |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 200 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 201 | def __hash__(self): |
| 202 | return hash(self.identity()) |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 203 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 204 | def __cmp__(self, other): |
| 205 | return self.identity() == other.identity() |
Craig Tiller | db21899 | 2016-01-05 09:28:46 -0800 | [diff] [blame] | 206 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 207 | def __repr__(self): |
| 208 | return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, |
| 209 | self.cmdline) |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 210 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 211 | def __str__(self): |
ncteisen | 0cd6cfe | 2017-12-11 16:56:44 -0800 | [diff] [blame] | 212 | return '%s: %s %s' % (self.shortname, ' '.join( |
| 213 | '%s=%s' % kv |
| 214 | for kv in self.environ.items()), ' '.join(self.cmdline)) |
Yong Ni | 35ee7e7 | 2017-04-26 17:45:45 -0700 | [diff] [blame] | 215 | |
Craig Tiller | 547db2b | 2015-01-30 14:08:39 -0800 | [diff] [blame] | 216 | |
Adele Zhou | e4c3561 | 2015-10-16 15:34:23 -0700 | [diff] [blame] | 217 | class JobResult(object): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 218 | |
| 219 | def __init__(self): |
| 220 | self.state = 'UNKNOWN' |
| 221 | self.returncode = -1 |
| 222 | self.elapsed_time = 0 |
| 223 | self.num_failures = 0 |
| 224 | self.retries = 0 |
| 225 | self.message = '' |
| 226 | self.cpu_estimated = 1 |
| 227 | self.cpu_measured = 1 |
Adele Zhou | e4c3561 | 2015-10-16 15:34:23 -0700 | [diff] [blame] | 228 | |
Craig Tiller | 9d5d803 | 2017-05-15 07:54:54 -0700 | [diff] [blame] | 229 | |
Craig Tiller | 9d5d803 | 2017-05-15 07:54:54 -0700 | [diff] [blame] | 230 | def read_from_start(f): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 231 | f.seek(0) |
| 232 | return f.read() |
Craig Tiller | 9d5d803 | 2017-05-15 07:54:54 -0700 | [diff] [blame] | 233 | |
| 234 | |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 235 | class Job(object): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 236 | """Manages one job.""" |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 237 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 238 | def __init__(self, |
| 239 | spec, |
| 240 | newline_on_success, |
| 241 | travis, |
| 242 | add_env, |
| 243 | quiet_success=False): |
| 244 | self._spec = spec |
| 245 | self._newline_on_success = newline_on_success |
| 246 | self._travis = travis |
| 247 | self._add_env = add_env.copy() |
| 248 | self._retries = 0 |
| 249 | self._timeout_retries = 0 |
| 250 | self._suppress_failure_message = False |
| 251 | self._quiet_success = quiet_success |
Jan Tattermusch | 68e27bf | 2016-12-16 14:09:03 +0100 | [diff] [blame] | 252 | if not self._quiet_success: |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 253 | message('START', spec.shortname, do_newline=self._travis) |
| 254 | self.result = JobResult() |
Craig Tiller | 3dc1e4f | 2015-09-25 11:46:56 -0700 | [diff] [blame] | 255 | self.start() |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 256 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 257 | def GetSpec(self): |
| 258 | return self._spec |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 259 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 260 | def start(self): |
| 261 | self._tempfile = tempfile.TemporaryFile() |
| 262 | env = dict(os.environ) |
| 263 | env.update(self._spec.environ) |
| 264 | env.update(self._add_env) |
| 265 | env = sanitized_environment(env) |
| 266 | self._start = time.time() |
| 267 | cmdline = self._spec.cmdline |
| 268 | # The Unix time command is finicky when used with MSBuild, so we don't use it |
| 269 | # with jobs that run MSBuild. |
| 270 | global measure_cpu_costs |
| 271 | if measure_cpu_costs and not 'vsprojects\\build' in cmdline[0]: |
| 272 | cmdline = ['time', '-p'] + cmdline |
| 273 | else: |
| 274 | measure_cpu_costs = False |
| 275 | try_start = lambda: subprocess.Popen(args=cmdline, |
| 276 | stderr=subprocess.STDOUT, |
| 277 | stdout=self._tempfile, |
| 278 | cwd=self._spec.cwd, |
| 279 | shell=self._spec.shell, |
| 280 | env=env) |
| 281 | delay = 0.3 |
| 282 | for i in range(0, 4): |
| 283 | try: |
| 284 | self._process = try_start() |
| 285 | break |
| 286 | except OSError: |
| 287 | message('WARNING', 'Failed to start %s, retrying in %f seconds' |
| 288 | % (self._spec.shortname, delay)) |
| 289 | time.sleep(delay) |
| 290 | delay *= 2 |
| 291 | else: |
| 292 | self._process = try_start() |
| 293 | self._state = _RUNNING |
| 294 | |
| 295 | def state(self): |
| 296 | """Poll current state of the job. Prints messages at completion.""" |
| 297 | |
| 298 | def stdout(self=self): |
| 299 | stdout = read_from_start(self._tempfile) |
| 300 | self.result.message = stdout[-_MAX_RESULT_SIZE:] |
| 301 | return stdout |
| 302 | |
| 303 | if self._state == _RUNNING and self._process.poll() is not None: |
| 304 | elapsed = time.time() - self._start |
| 305 | self.result.elapsed_time = elapsed |
| 306 | if self._process.returncode != 0: |
| 307 | if self._retries < self._spec.flake_retries: |
| 308 | message( |
| 309 | 'FLAKE', |
| 310 | '%s [ret=%d, pid=%d]' % |
| 311 | (self._spec.shortname, self._process.returncode, |
| 312 | self._process.pid), |
| 313 | stdout(), |
| 314 | do_newline=True) |
| 315 | self._retries += 1 |
| 316 | self.result.num_failures += 1 |
| 317 | self.result.retries = self._timeout_retries + self._retries |
| 318 | # NOTE: job is restarted regardless of jobset's max_time setting |
| 319 | self.start() |
| 320 | else: |
| 321 | self._state = _FAILURE |
| 322 | if not self._suppress_failure_message: |
| 323 | message( |
| 324 | 'FAILED', |
| 325 | '%s [ret=%d, pid=%d, time=%.1fsec]' % |
| 326 | (self._spec.shortname, self._process.returncode, |
| 327 | self._process.pid, elapsed), |
| 328 | stdout(), |
| 329 | do_newline=True) |
| 330 | self.result.state = 'FAILED' |
| 331 | self.result.num_failures += 1 |
| 332 | self.result.returncode = self._process.returncode |
| 333 | else: |
| 334 | self._state = _SUCCESS |
| 335 | measurement = '' |
| 336 | if measure_cpu_costs: |
| 337 | m = re.search( |
| 338 | r'real\s+([0-9.]+)\nuser\s+([0-9.]+)\nsys\s+([0-9.]+)', |
| 339 | stdout()) |
| 340 | real = float(m.group(1)) |
| 341 | user = float(m.group(2)) |
| 342 | sys = float(m.group(3)) |
| 343 | if real > 0.5: |
| 344 | cores = (user + sys) / real |
| 345 | self.result.cpu_measured = float('%.01f' % cores) |
| 346 | self.result.cpu_estimated = float('%.01f' % |
| 347 | self._spec.cpu_cost) |
| 348 | measurement = '; cpu_cost=%.01f; estimated=%.01f' % ( |
| 349 | self.result.cpu_measured, self.result.cpu_estimated) |
| 350 | if not self._quiet_success: |
| 351 | message( |
| 352 | 'PASSED', |
| 353 | '%s [time=%.1fsec, retries=%d:%d%s]' % |
| 354 | (self._spec.shortname, elapsed, self._retries, |
| 355 | self._timeout_retries, measurement), |
| 356 | stdout() if self._spec.verbose_success else None, |
| 357 | do_newline=self._newline_on_success or self._travis) |
| 358 | self.result.state = 'PASSED' |
| 359 | elif (self._state == _RUNNING and |
| 360 | self._spec.timeout_seconds is not None and |
| 361 | time.time() - self._start > self._spec.timeout_seconds): |
| 362 | elapsed = time.time() - self._start |
| 363 | self.result.elapsed_time = elapsed |
| 364 | if self._timeout_retries < self._spec.timeout_retries: |
| 365 | message( |
| 366 | 'TIMEOUT_FLAKE', |
| 367 | '%s [pid=%d]' % (self._spec.shortname, self._process.pid), |
| 368 | stdout(), |
| 369 | do_newline=True) |
| 370 | self._timeout_retries += 1 |
| 371 | self.result.num_failures += 1 |
| 372 | self.result.retries = self._timeout_retries + self._retries |
| 373 | if self._spec.kill_handler: |
| 374 | self._spec.kill_handler(self) |
| 375 | self._process.terminate() |
| 376 | # NOTE: job is restarted regardless of jobset's max_time setting |
| 377 | self.start() |
| 378 | else: |
| 379 | message( |
| 380 | 'TIMEOUT', |
| 381 | '%s [pid=%d, time=%.1fsec]' % |
| 382 | (self._spec.shortname, self._process.pid, elapsed), |
| 383 | stdout(), |
| 384 | do_newline=True) |
| 385 | self.kill() |
| 386 | self.result.state = 'TIMEOUT' |
| 387 | self.result.num_failures += 1 |
| 388 | return self._state |
| 389 | |
| 390 | def kill(self): |
| 391 | if self._state == _RUNNING: |
| 392 | self._state = _KILLED |
| 393 | if self._spec.kill_handler: |
| 394 | self._spec.kill_handler(self) |
| 395 | self._process.terminate() |
| 396 | |
| 397 | def suppress_failure_message(self): |
| 398 | self._suppress_failure_message = True |
Craig Tiller | db21899 | 2016-01-05 09:28:46 -0800 | [diff] [blame] | 399 | |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 400 | |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 401 | class Jobset(object): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 402 | """Manages one run of jobs.""" |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 403 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 404 | def __init__(self, check_cancelled, maxjobs, maxjobs_cpu_agnostic, |
| 405 | newline_on_success, travis, stop_on_failure, add_env, |
| 406 | quiet_success, max_time): |
| 407 | self._running = set() |
| 408 | self._check_cancelled = check_cancelled |
| 409 | self._cancelled = False |
| 410 | self._failures = 0 |
| 411 | self._completed = 0 |
| 412 | self._maxjobs = maxjobs |
| 413 | self._maxjobs_cpu_agnostic = maxjobs_cpu_agnostic |
| 414 | self._newline_on_success = newline_on_success |
| 415 | self._travis = travis |
| 416 | self._stop_on_failure = stop_on_failure |
| 417 | self._add_env = add_env |
| 418 | self._quiet_success = quiet_success |
| 419 | self._max_time = max_time |
| 420 | self.resultset = {} |
| 421 | self._remaining = None |
| 422 | self._start_time = time.time() |
Craig Tiller | 6364dcb | 2015-11-24 16:29:06 -0800 | [diff] [blame] | 423 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 424 | def set_remaining(self, remaining): |
| 425 | self._remaining = remaining |
Craig Tiller | 6364dcb | 2015-11-24 16:29:06 -0800 | [diff] [blame] | 426 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 427 | def get_num_failures(self): |
| 428 | return self._failures |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 429 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 430 | def cpu_cost(self): |
| 431 | c = 0 |
| 432 | for job in self._running: |
| 433 | c += job._spec.cpu_cost |
| 434 | return c |
Craig Tiller | 56c6b6a | 2016-01-20 08:27:37 -0800 | [diff] [blame] | 435 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 436 | def start(self, spec): |
| 437 | """Start a job. Return True on success, False on failure.""" |
| 438 | while True: |
| 439 | if self._max_time > 0 and time.time( |
| 440 | ) - self._start_time > self._max_time: |
| 441 | skipped_job_result = JobResult() |
| 442 | skipped_job_result.state = 'SKIPPED' |
| 443 | message('SKIPPED', spec.shortname, do_newline=True) |
| 444 | self.resultset[spec.shortname] = [skipped_job_result] |
| 445 | return True |
| 446 | if self.cancelled(): return False |
| 447 | current_cpu_cost = self.cpu_cost() |
| 448 | if current_cpu_cost == 0: break |
| 449 | if current_cpu_cost + spec.cpu_cost <= self._maxjobs: |
| 450 | if len(self._running) < self._maxjobs_cpu_agnostic: |
| 451 | break |
| 452 | self.reap(spec.shortname, spec.cpu_cost) |
| 453 | if self.cancelled(): return False |
| 454 | job = Job(spec, self._newline_on_success, self._travis, self._add_env, |
| 455 | self._quiet_success) |
| 456 | self._running.add(job) |
| 457 | if job.GetSpec().shortname not in self.resultset: |
| 458 | self.resultset[job.GetSpec().shortname] = [] |
Craig Tiller | a1ac2a1 | 2017-04-21 07:20:38 -0700 | [diff] [blame] | 459 | return True |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 460 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 461 | def reap(self, waiting_for=None, waiting_for_cost=None): |
| 462 | """Collect the dead jobs.""" |
| 463 | while self._running: |
| 464 | dead = set() |
Craig Tiller | 533b1a2 | 2015-05-29 08:41:29 -0700 | [diff] [blame] | 465 | for job in self._running: |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 466 | st = eintr_be_gone(lambda: job.state()) |
| 467 | if st == _RUNNING: continue |
| 468 | if st == _FAILURE or st == _KILLED: |
| 469 | self._failures += 1 |
| 470 | if self._stop_on_failure: |
| 471 | self._cancelled = True |
| 472 | for job in self._running: |
| 473 | job.kill() |
| 474 | dead.add(job) |
| 475 | break |
| 476 | for job in dead: |
| 477 | self._completed += 1 |
| 478 | if not self._quiet_success or job.result.state != 'PASSED': |
| 479 | self.resultset[job.GetSpec().shortname].append(job.result) |
| 480 | self._running.remove(job) |
| 481 | if dead: return |
| 482 | if not self._travis and platform_string() != 'windows': |
| 483 | rstr = '' if self._remaining is None else '%d queued, ' % self._remaining |
| 484 | if self._remaining is not None and self._completed > 0: |
| 485 | now = time.time() |
| 486 | sofar = now - self._start_time |
| 487 | remaining = sofar / self._completed * ( |
| 488 | self._remaining + len(self._running)) |
| 489 | rstr = 'ETA %.1f sec; %s' % (remaining, rstr) |
| 490 | if waiting_for is not None: |
| 491 | wstr = ' next: %s @ %.2f cpu' % (waiting_for, |
| 492 | waiting_for_cost) |
| 493 | else: |
| 494 | wstr = '' |
| 495 | message( |
| 496 | 'WAITING', |
| 497 | '%s%d jobs running, %d complete, %d failed (load %.2f)%s' % |
| 498 | (rstr, len(self._running), self._completed, self._failures, |
| 499 | self.cpu_cost(), wstr)) |
| 500 | if platform_string() == 'windows': |
| 501 | time.sleep(0.1) |
| 502 | else: |
| 503 | signal.alarm(10) |
| 504 | signal.pause() |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 505 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 506 | def cancelled(self): |
| 507 | """Poll for cancellation.""" |
| 508 | if self._cancelled: return True |
| 509 | if not self._check_cancelled(): return False |
| 510 | for job in self._running: |
| 511 | job.kill() |
| 512 | self._cancelled = True |
| 513 | return True |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 514 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 515 | def finish(self): |
| 516 | while self._running: |
| 517 | if self.cancelled(): pass # poll cancellation |
| 518 | self.reap() |
| 519 | if platform_string() != 'windows': |
| 520 | signal.alarm(0) |
| 521 | return not self.cancelled() and self._failures == 0 |
Nicolas Noble | ddef246 | 2015-01-06 18:08:25 -0800 | [diff] [blame] | 522 | |
| 523 | |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 524 | def _never_cancelled(): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 525 | return False |
ctiller | 3040cb7 | 2015-01-07 12:13:17 -0800 | [diff] [blame] | 526 | |
| 527 | |
Craig Tiller | 6364dcb | 2015-11-24 16:29:06 -0800 | [diff] [blame] | 528 | def tag_remaining(xs): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 529 | staging = [] |
| 530 | for x in xs: |
| 531 | staging.append(x) |
| 532 | if len(staging) > 5000: |
| 533 | yield (staging.pop(0), None) |
| 534 | n = len(staging) |
| 535 | for i, x in enumerate(staging): |
| 536 | yield (x, n - i - 1) |
Craig Tiller | 6364dcb | 2015-11-24 16:29:06 -0800 | [diff] [blame] | 537 | |
| 538 | |
Nicolas Noble | b09078f | 2015-01-14 18:06:05 -0800 | [diff] [blame] | 539 | def run(cmdlines, |
| 540 | check_cancelled=_never_cancelled, |
| 541 | maxjobs=None, |
Alexander Polcyn | dbfcd45 | 2017-10-01 15:34:29 -0700 | [diff] [blame] | 542 | maxjobs_cpu_agnostic=None, |
Craig Tiller | 7173518 | 2015-01-15 17:07:13 -0800 | [diff] [blame] | 543 | newline_on_success=False, |
Nicolas "Pixel" Noble | a7df3f9 | 2015-02-26 22:07:04 +0100 | [diff] [blame] | 544 | travis=False, |
David Garcia Quintas | e90cd37 | 2015-05-31 18:15:26 -0700 | [diff] [blame] | 545 | infinite_runs=False, |
Craig Tiller | 533b1a2 | 2015-05-29 08:41:29 -0700 | [diff] [blame] | 546 | stop_on_failure=False, |
Matt Kwong | 5c691c6 | 2016-10-20 17:11:18 -0700 | [diff] [blame] | 547 | add_env={}, |
Jan Tattermusch | 68e27bf | 2016-12-16 14:09:03 +0100 | [diff] [blame] | 548 | skip_jobs=False, |
Craig Tiller | a1ac2a1 | 2017-04-21 07:20:38 -0700 | [diff] [blame] | 549 | quiet_success=False, |
Matt Kwong | 69ce380 | 2017-09-12 13:30:51 -0700 | [diff] [blame] | 550 | max_time=-1): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 551 | if skip_jobs: |
| 552 | resultset = {} |
| 553 | skipped_job_result = JobResult() |
| 554 | skipped_job_result.state = 'SKIPPED' |
| 555 | for job in cmdlines: |
| 556 | message('SKIPPED', job.shortname, do_newline=True) |
| 557 | resultset[job.shortname] = [skipped_job_result] |
| 558 | return 0, resultset |
| 559 | js = Jobset(check_cancelled, maxjobs if maxjobs is not None else |
| 560 | _DEFAULT_MAX_JOBS, maxjobs_cpu_agnostic |
| 561 | if maxjobs_cpu_agnostic is not None else _DEFAULT_MAX_JOBS, |
| 562 | newline_on_success, travis, stop_on_failure, add_env, |
| 563 | quiet_success, max_time) |
| 564 | for cmdline, remaining in tag_remaining(cmdlines): |
| 565 | if not js.start(cmdline): |
| 566 | break |
| 567 | if remaining is not None: |
| 568 | js.set_remaining(remaining) |
| 569 | js.finish() |
| 570 | return js.get_num_failures(), js.resultset |