blob: e5e778a3f1642f8c969ba68704b4c629d85adeef [file] [log] [blame]
Craig Tillerc2c79212015-02-16 12:00:01 -08001# Copyright 2015, Google Inc.
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8# * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10# * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14# * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
Nicolas Nobleddef2462015-01-06 18:08:25 -080030"""Run a group of subprocesses and then finish."""
31
Craig Tiller71735182015-01-15 17:07:13 -080032import hashlib
Nicolas Nobleddef2462015-01-06 18:08:25 -080033import multiprocessing
Craig Tiller71735182015-01-15 17:07:13 -080034import os
Craig Tiller5058c692015-04-08 09:42:04 -070035import platform
Craig Tiller336ad502015-02-24 14:46:02 -080036import signal
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +020037import string
Nicolas Nobleddef2462015-01-06 18:08:25 -080038import subprocess
39import sys
ctiller3040cb72015-01-07 12:13:17 -080040import tempfile
41import time
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +020042import xml.etree.cElementTree as ET
Nicolas Nobleddef2462015-01-06 18:08:25 -080043
ctiller3040cb72015-01-07 12:13:17 -080044
ctiller94e5dde2015-01-09 10:41:59 -080045_DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
Nicolas Nobleddef2462015-01-06 18:08:25 -080046
47
Craig Tiller336ad502015-02-24 14:46:02 -080048# setup a signal handler so that signal.pause registers 'something'
49# when a child finishes
50# not using futures and threading to avoid a dependency on subprocess32
Craig Tiller5058c692015-04-08 09:42:04 -070051if platform.system() == "Windows":
52 pass
53else:
54 have_alarm = False
55 def alarm_handler(unused_signum, unused_frame):
56 global have_alarm
57 have_alarm = False
58
59 signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None)
60 signal.signal(signal.SIGALRM, alarm_handler)
Craig Tiller336ad502015-02-24 14:46:02 -080061
62
ctiller3040cb72015-01-07 12:13:17 -080063_SUCCESS = object()
64_FAILURE = object()
65_RUNNING = object()
66_KILLED = object()
67
68
Craig Tiller3b083062015-01-12 13:51:28 -080069_COLORS = {
Nicolas Noble044db742015-01-14 16:57:24 -080070 'red': [ 31, 0 ],
71 'green': [ 32, 0 ],
72 'yellow': [ 33, 0 ],
73 'lightgray': [ 37, 0],
74 'gray': [ 30, 1 ],
Craig Tiller3b083062015-01-12 13:51:28 -080075 }
76
77
78_BEGINNING_OF_LINE = '\x1b[0G'
79_CLEAR_LINE = '\x1b[2K'
80
81
82_TAG_COLOR = {
83 'FAILED': 'red',
Masood Malekghassemie5f70022015-06-29 09:20:26 -070084 'WARNING': 'yellow',
Craig Tillere1d0d1c2015-02-27 08:54:23 -080085 'TIMEOUT': 'red',
Craig Tiller3b083062015-01-12 13:51:28 -080086 'PASSED': 'green',
Nicolas Noble044db742015-01-14 16:57:24 -080087 'START': 'gray',
Craig Tiller3b083062015-01-12 13:51:28 -080088 'WAITING': 'yellow',
Nicolas Noble044db742015-01-14 16:57:24 -080089 'SUCCESS': 'green',
90 'IDLE': 'gray',
Craig Tiller3b083062015-01-12 13:51:28 -080091 }
92
93
Nicolas "Pixel" Noble99768ac2015-05-13 02:34:06 +020094def message(tag, msg, explanatory_text=None, do_newline=False):
95 if message.old_tag == tag and message.old_msg == msg and not explanatory_text:
96 return
97 message.old_tag = tag
98 message.old_msg = msg
vjpaia29d2d72015-07-08 10:31:15 -070099 if platform.system() == 'Windows' or not sys.stdout.isatty():
Craig Tiller5058c692015-04-08 09:42:04 -0700100 if explanatory_text:
101 print explanatory_text
Nicolas "Pixel" Noble99768ac2015-05-13 02:34:06 +0200102 print '%s: %s' % (tag, msg)
Craig Tiller5058c692015-04-08 09:42:04 -0700103 return
Craig Tiller23d2f3f2015-02-24 15:23:32 -0800104 try:
vjpaia29d2d72015-07-08 10:31:15 -0700105 sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % (
106 _BEGINNING_OF_LINE,
107 _CLEAR_LINE,
108 '\n%s' % explanatory_text if explanatory_text is not None else '',
109 _COLORS[_TAG_COLOR[tag]][1],
110 _COLORS[_TAG_COLOR[tag]][0],
111 tag,
112 msg,
113 '\n' if do_newline or explanatory_text is not None else ''))
Craig Tiller23d2f3f2015-02-24 15:23:32 -0800114 sys.stdout.flush()
115 except:
116 pass
Craig Tiller3b083062015-01-12 13:51:28 -0800117
Nicolas "Pixel" Noble99768ac2015-05-13 02:34:06 +0200118message.old_tag = ""
119message.old_msg = ""
Craig Tiller3b083062015-01-12 13:51:28 -0800120
Craig Tiller71735182015-01-15 17:07:13 -0800121def which(filename):
122 if '/' in filename:
123 return filename
124 for path in os.environ['PATH'].split(os.pathsep):
125 if os.path.exists(os.path.join(path, filename)):
126 return os.path.join(path, filename)
127 raise Exception('%s not found' % filename)
128
129
Craig Tiller547db2b2015-01-30 14:08:39 -0800130class JobSpec(object):
131 """Specifies what to run for a job."""
132
Jan Tattermusch725835a2015-08-01 21:02:35 -0700133 def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
134 cwd=None, shell=False, timeout_seconds=900):
Craig Tiller547db2b2015-01-30 14:08:39 -0800135 """
136 Arguments:
137 cmdline: a list of arguments to pass as the command line
138 environ: a dictionary of environment variables to set in the child process
139 hash_targets: which files to include in the hash representing the jobs version
140 (or empty, indicating the job should not be hashed)
141 """
murgatroid99132ce6a2015-03-04 17:29:14 -0800142 if environ is None:
143 environ = {}
144 if hash_targets is None:
145 hash_targets = []
Craig Tiller547db2b2015-01-30 14:08:39 -0800146 self.cmdline = cmdline
147 self.environ = environ
148 self.shortname = cmdline[0] if shortname is None else shortname
149 self.hash_targets = hash_targets or []
Craig Tiller5058c692015-04-08 09:42:04 -0700150 self.cwd = cwd
Jan Tattermusche8243592015-04-17 14:14:01 -0700151 self.shell = shell
Jan Tattermusch725835a2015-08-01 21:02:35 -0700152 self.timeout_seconds = timeout_seconds
Craig Tiller547db2b2015-01-30 14:08:39 -0800153
154 def identity(self):
155 return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
156
157 def __hash__(self):
158 return hash(self.identity())
159
160 def __cmp__(self, other):
161 return self.identity() == other.identity()
162
163
ctiller3040cb72015-01-07 12:13:17 -0800164class Job(object):
165 """Manages one job."""
166
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200167 def __init__(self, spec, bin_hash, newline_on_success, travis, xml_report):
Craig Tiller547db2b2015-01-30 14:08:39 -0800168 self._spec = spec
Craig Tiller71735182015-01-15 17:07:13 -0800169 self._bin_hash = bin_hash
ctiller3040cb72015-01-07 12:13:17 -0800170 self._tempfile = tempfile.TemporaryFile()
Craig Tiller547db2b2015-01-30 14:08:39 -0800171 env = os.environ.copy()
172 for k, v in spec.environ.iteritems():
173 env[k] = v
Craig Tiller9d6139a2015-02-26 15:24:43 -0800174 self._start = time.time()
Craig Tiller547db2b2015-01-30 14:08:39 -0800175 self._process = subprocess.Popen(args=spec.cmdline,
ctiller3040cb72015-01-07 12:13:17 -0800176 stderr=subprocess.STDOUT,
Craig Tiller547db2b2015-01-30 14:08:39 -0800177 stdout=self._tempfile,
Craig Tiller5058c692015-04-08 09:42:04 -0700178 cwd=spec.cwd,
Jan Tattermusche8243592015-04-17 14:14:01 -0700179 shell=spec.shell,
Craig Tiller547db2b2015-01-30 14:08:39 -0800180 env=env)
ctiller3040cb72015-01-07 12:13:17 -0800181 self._state = _RUNNING
Nicolas Noble044db742015-01-14 16:57:24 -0800182 self._newline_on_success = newline_on_success
Nicolas "Pixel" Noblea7df3f92015-02-26 22:07:04 +0100183 self._travis = travis
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200184 self._xml_test = ET.SubElement(xml_report, 'testcase',
185 name=self._spec.shortname) if xml_report is not None else None
Craig Tillerb84728d2015-02-26 15:40:39 -0800186 message('START', spec.shortname, do_newline=self._travis)
ctiller3040cb72015-01-07 12:13:17 -0800187
Craig Tiller71735182015-01-15 17:07:13 -0800188 def state(self, update_cache):
ctiller3040cb72015-01-07 12:13:17 -0800189 """Poll current state of the job. Prints messages at completion."""
190 if self._state == _RUNNING and self._process.poll() is not None:
Craig Tiller9d6139a2015-02-26 15:24:43 -0800191 elapsed = time.time() - self._start
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200192 self._tempfile.seek(0)
193 stdout = self._tempfile.read()
194 filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
195 if self._xml_test is not None:
196 self._xml_test.set('time', str(elapsed))
197 ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
ctiller3040cb72015-01-07 12:13:17 -0800198 if self._process.returncode != 0:
199 self._state = _FAILURE
Craig Tillerd0ffe142015-05-19 21:51:13 -0700200 message('FAILED', '%s [ret=%d, pid=%d]' % (
201 self._spec.shortname, self._process.returncode, self._process.pid),
202 stdout, do_newline=True)
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200203 if self._xml_test is not None:
204 ET.SubElement(self._xml_test, 'failure', message='Failure').text
ctiller3040cb72015-01-07 12:13:17 -0800205 else:
206 self._state = _SUCCESS
Craig Tiller9d6139a2015-02-26 15:24:43 -0800207 message('PASSED', '%s [time=%.1fsec]' % (self._spec.shortname, elapsed),
Nicolas "Pixel" Noblea7df3f92015-02-26 22:07:04 +0100208 do_newline=self._newline_on_success or self._travis)
Craig Tiller547db2b2015-01-30 14:08:39 -0800209 if self._bin_hash:
210 update_cache.finished(self._spec.identity(), self._bin_hash)
Jan Tattermusch725835a2015-08-01 21:02:35 -0700211 elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
Craig Tiller84216782015-05-12 09:43:54 -0700212 self._tempfile.seek(0)
213 stdout = self._tempfile.read()
Nicolas "Pixel" Noblef716c0c2015-07-12 01:26:17 +0200214 filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
Craig Tiller84216782015-05-12 09:43:54 -0700215 message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
Craig Tiller9b3cc742015-02-26 22:25:03 -0800216 self.kill()
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200217 if self._xml_test is not None:
Nicolas "Pixel" Noblef716c0c2015-07-12 01:26:17 +0200218 ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200219 ET.SubElement(self._xml_test, 'error', message='Timeout')
ctiller3040cb72015-01-07 12:13:17 -0800220 return self._state
221
222 def kill(self):
223 if self._state == _RUNNING:
224 self._state = _KILLED
225 self._process.terminate()
226
227
Nicolas Nobleddef2462015-01-06 18:08:25 -0800228class Jobset(object):
229 """Manages one run of jobs."""
230
Craig Tiller533b1a22015-05-29 08:41:29 -0700231 def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200232 stop_on_failure, cache, xml_report):
ctiller3040cb72015-01-07 12:13:17 -0800233 self._running = set()
234 self._check_cancelled = check_cancelled
235 self._cancelled = False
Nicolas Nobleddef2462015-01-06 18:08:25 -0800236 self._failures = 0
Craig Tiller738c3342015-01-12 14:28:33 -0800237 self._completed = 0
ctiller94e5dde2015-01-09 10:41:59 -0800238 self._maxjobs = maxjobs
Nicolas Noble044db742015-01-14 16:57:24 -0800239 self._newline_on_success = newline_on_success
Nicolas "Pixel" Noblea7df3f92015-02-26 22:07:04 +0100240 self._travis = travis
Craig Tiller71735182015-01-15 17:07:13 -0800241 self._cache = cache
Craig Tiller533b1a22015-05-29 08:41:29 -0700242 self._stop_on_failure = stop_on_failure
Craig Tiller74e770d2015-06-11 09:38:09 -0700243 self._hashes = {}
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200244 self._xml_report = xml_report
Nicolas Nobleddef2462015-01-06 18:08:25 -0800245
Craig Tiller547db2b2015-01-30 14:08:39 -0800246 def start(self, spec):
ctiller3040cb72015-01-07 12:13:17 -0800247 """Start a job. Return True on success, False on failure."""
ctiller94e5dde2015-01-09 10:41:59 -0800248 while len(self._running) >= self._maxjobs:
ctiller3040cb72015-01-07 12:13:17 -0800249 if self.cancelled(): return False
250 self.reap()
251 if self.cancelled(): return False
Craig Tiller547db2b2015-01-30 14:08:39 -0800252 if spec.hash_targets:
Craig Tiller74e770d2015-06-11 09:38:09 -0700253 if spec.identity() in self._hashes:
254 bin_hash = self._hashes[spec.identity()]
255 else:
256 bin_hash = hashlib.sha1()
257 for fn in spec.hash_targets:
258 with open(which(fn)) as f:
259 bin_hash.update(f.read())
260 bin_hash = bin_hash.hexdigest()
261 self._hashes[spec.identity()] = bin_hash
Craig Tiller547db2b2015-01-30 14:08:39 -0800262 should_run = self._cache.should_run(spec.identity(), bin_hash)
263 else:
264 bin_hash = None
265 should_run = True
266 if should_run:
Craig Tiller5058c692015-04-08 09:42:04 -0700267 try:
268 self._running.add(Job(spec,
269 bin_hash,
270 self._newline_on_success,
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200271 self._travis,
272 self._xml_report))
Craig Tiller5058c692015-04-08 09:42:04 -0700273 except:
274 message('FAILED', spec.shortname)
275 self._cancelled = True
276 return False
ctiller3040cb72015-01-07 12:13:17 -0800277 return True
Nicolas Nobleddef2462015-01-06 18:08:25 -0800278
ctiller3040cb72015-01-07 12:13:17 -0800279 def reap(self):
280 """Collect the dead jobs."""
281 while self._running:
282 dead = set()
283 for job in self._running:
Craig Tiller71735182015-01-15 17:07:13 -0800284 st = job.state(self._cache)
ctiller3040cb72015-01-07 12:13:17 -0800285 if st == _RUNNING: continue
Craig Tiller533b1a22015-05-29 08:41:29 -0700286 if st == _FAILURE or st == _KILLED:
287 self._failures += 1
288 if self._stop_on_failure:
289 self._cancelled = True
290 for job in self._running:
291 job.kill()
ctiller3040cb72015-01-07 12:13:17 -0800292 dead.add(job)
Craig Tiller74e770d2015-06-11 09:38:09 -0700293 break
ctiller3040cb72015-01-07 12:13:17 -0800294 for job in dead:
Craig Tiller738c3342015-01-12 14:28:33 -0800295 self._completed += 1
ctiller3040cb72015-01-07 12:13:17 -0800296 self._running.remove(job)
Craig Tiller3b083062015-01-12 13:51:28 -0800297 if dead: return
Nicolas "Pixel" Noblea7df3f92015-02-26 22:07:04 +0100298 if (not self._travis):
299 message('WAITING', '%d jobs running, %d complete, %d failed' % (
300 len(self._running), self._completed, self._failures))
Craig Tiller5058c692015-04-08 09:42:04 -0700301 if platform.system() == 'Windows':
302 time.sleep(0.1)
303 else:
304 global have_alarm
305 if not have_alarm:
306 have_alarm = True
307 signal.alarm(10)
308 signal.pause()
ctiller3040cb72015-01-07 12:13:17 -0800309
310 def cancelled(self):
311 """Poll for cancellation."""
312 if self._cancelled: return True
313 if not self._check_cancelled(): return False
314 for job in self._running:
315 job.kill()
316 self._cancelled = True
317 return True
318
319 def finish(self):
320 while self._running:
321 if self.cancelled(): pass # poll cancellation
322 self.reap()
323 return not self.cancelled() and self._failures == 0
Nicolas Nobleddef2462015-01-06 18:08:25 -0800324
325
ctiller3040cb72015-01-07 12:13:17 -0800326def _never_cancelled():
327 return False
328
329
Craig Tiller71735182015-01-15 17:07:13 -0800330# cache class that caches nothing
331class NoCache(object):
332 def should_run(self, cmdline, bin_hash):
333 return True
334
335 def finished(self, cmdline, bin_hash):
336 pass
337
338
Nicolas Nobleb09078f2015-01-14 18:06:05 -0800339def run(cmdlines,
340 check_cancelled=_never_cancelled,
341 maxjobs=None,
Craig Tiller71735182015-01-15 17:07:13 -0800342 newline_on_success=False,
Nicolas "Pixel" Noblea7df3f92015-02-26 22:07:04 +0100343 travis=False,
David Garcia Quintase90cd372015-05-31 18:15:26 -0700344 infinite_runs=False,
Craig Tiller533b1a22015-05-29 08:41:29 -0700345 stop_on_failure=False,
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200346 cache=None,
347 xml_report=None):
ctiller94e5dde2015-01-09 10:41:59 -0800348 js = Jobset(check_cancelled,
Nicolas Noble044db742015-01-14 16:57:24 -0800349 maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
Craig Tiller533b1a22015-05-29 08:41:29 -0700350 newline_on_success, travis, stop_on_failure,
Nicolas "Pixel" Noble5937b5b2015-06-26 02:04:12 +0200351 cache if cache is not None else NoCache(),
352 xml_report)
Craig Tillerb84728d2015-02-26 15:40:39 -0800353 for cmdline in cmdlines:
ctiller3040cb72015-01-07 12:13:17 -0800354 if not js.start(cmdline):
355 break
356 return js.finish()