blob: f9e91370cf353e9bd69071b89eaebdba1a45c1dd [file] [log] [blame]
mblighc86b0b42006-07-28 17:35:28 +00001"""The main job wrapper
mbligha2508052006-05-28 21:29:53 +00002
mblighc86b0b42006-07-28 17:35:28 +00003This is the core infrastructure.
4"""
5
6__author__ = """Copyright Andy Whitcroft, Martin J. Bligh 2006"""
mbligha2508052006-05-28 21:29:53 +00007
mbligh8f243ec2006-10-10 05:55:49 +00008# standard stuff
apw8fef4ac2006-10-10 22:53:37 +00009import os, sys, re, pickle, shutil
mbligh8f243ec2006-10-10 05:55:49 +000010# autotest stuff
mblighf4c35322006-03-13 01:01:10 +000011from autotest_utils import *
apw8fef4ac2006-10-10 22:53:37 +000012from parallel import *
mbligh8baa2ea2006-12-17 23:01:24 +000013import kernel, xen, test, profilers, barrier, filesystem, fd_stack, boottool
apw059e1b12006-10-12 17:18:26 +000014import harness, config
mblighf4c35322006-03-13 01:01:10 +000015
16class job:
mblighc86b0b42006-07-28 17:35:28 +000017 """The actual job against which we do everything.
18
19 Properties:
mbligh72b88fc2006-12-16 18:41:35 +000020 autodir
mblighc86b0b42006-07-28 17:35:28 +000021 The top level autotest directory (/usr/local/autotest).
22 Comes from os.environ['AUTODIR'].
mbligh72b88fc2006-12-16 18:41:35 +000023 bindir
mblighc86b0b42006-07-28 17:35:28 +000024 <autodir>/bin/
mbligh72b88fc2006-12-16 18:41:35 +000025 testdir
mblighc86b0b42006-07-28 17:35:28 +000026 <autodir>/tests/
27 profdir
28 <autodir>/profilers/
29 tmpdir
30 <autodir>/tmp/
31 resultdir
32 <autodir>/results/<jobtag>
33 stdout
34 fd_stack object for stdout
35 stderr
36 fd_stack object for stderr
37 profilers
38 the profilers object for this job
apw504a7dd2006-10-12 17:18:37 +000039 harness
40 the server harness object for this job
apw059e1b12006-10-12 17:18:26 +000041 config
42 the job configuration for this job
mblighc86b0b42006-07-28 17:35:28 +000043 """
44
mbligh362ab3d2007-08-30 11:24:04 +000045 def __init__(self, control, jobtag, cont, harness_type=None):
mblighc86b0b42006-07-28 17:35:28 +000046 """
47 control
48 The control file (pathname of)
49 jobtag
50 The job tag string (eg "default")
apw96da1a42006-11-02 00:23:18 +000051 cont
52 If this is the continuation of this job
apwe68a7132006-12-01 11:21:37 +000053 harness_type
54 An alternative server harness
mblighc86b0b42006-07-28 17:35:28 +000055 """
mblighf4c35322006-03-13 01:01:10 +000056 self.autodir = os.environ['AUTODIR']
mbligh06743772006-05-18 21:30:19 +000057 self.bindir = self.autodir + '/bin'
mbligh82641862006-04-23 06:21:36 +000058 self.testdir = self.autodir + '/tests'
mbligha2508052006-05-28 21:29:53 +000059 self.profdir = self.autodir + '/profilers'
mblighf4c35322006-03-13 01:01:10 +000060 self.tmpdir = self.autodir + '/tmp'
mbligh24f7da02006-04-23 06:32:18 +000061 self.resultdir = self.autodir + '/results/' + jobtag
mbligha2508052006-05-28 21:29:53 +000062
apw96da1a42006-11-02 00:23:18 +000063 if not cont:
64 if os.path.exists(self.tmpdir):
65 system('rm -rf ' + self.tmpdir)
66 os.mkdir(self.tmpdir)
67
mblighfbfb77d2007-02-15 18:54:03 +000068 if not os.path.exists(self.autodir + '/results'):
69 os.mkdir(self.autodir + '/results')
70
apw96da1a42006-11-02 00:23:18 +000071 if os.path.exists(self.resultdir):
72 system('rm -rf ' + self.resultdir)
73 os.mkdir(self.resultdir)
74
75 os.mkdir(self.resultdir + "/debug")
76 os.mkdir(self.resultdir + "/analysis")
77 os.mkdir(self.resultdir + "/sysinfo")
mblighd9223fc2006-11-26 17:19:54 +000078 shutil.copyfile(control, self.resultdir + "/control")
mbligh4b089662006-06-14 22:34:58 +000079
apwecf41b72006-03-31 14:00:55 +000080 self.control = control
mblighf4c35322006-03-13 01:01:10 +000081 self.jobtab = jobtag
82
mbligh56f1fbb2006-10-01 15:10:56 +000083 self.stdout = fd_stack.fd_stack(1, sys.stdout)
84 self.stderr = fd_stack.fd_stack(2, sys.stderr)
mbligh88ab90f2007-08-29 15:52:49 +000085 self.record_prefix = ''
mblighf4c35322006-03-13 01:01:10 +000086
apw059e1b12006-10-12 17:18:26 +000087 self.config = config.config(self)
88
apwd27e55f2006-12-01 11:22:08 +000089 self.harness = harness.select(harness_type, self)
90
mbligha35553b2006-04-23 15:52:25 +000091 self.profilers = profilers.profilers(self)
mbligh72905562006-05-25 01:30:49 +000092
mblighcaa605c2006-10-02 00:37:35 +000093 try:
apw90154af2006-12-01 11:23:36 +000094 tool = self.config_get('boottool.executable')
95 self.bootloader = boottool.boottool(tool)
mblighcaa605c2006-10-02 00:37:35 +000096 except:
97 pass
98
mbligh72b88fc2006-12-16 18:41:35 +000099 pwd = os.getcwd()
mbligh06743772006-05-18 21:30:19 +0000100 os.chdir(self.resultdir + "/sysinfo")
101 system(self.bindir + '/sysinfo.py')
mblighe21abaf2007-07-26 16:17:30 +0000102 system('dmesg -c > dmesg', ignorestatus=1)
mbligh06743772006-05-18 21:30:19 +0000103 os.chdir(pwd)
mbligh3a6d6ca2006-04-23 15:50:24 +0000104
apw357f50f2006-12-01 11:22:39 +0000105 self.harness.run_start()
106
mbligh0692e472007-08-30 16:07:53 +0000107
108 def relative_path(self, path):
109 """\
110 Return a patch relative to the job results directory
111 """
mbligh1c250ca2007-08-30 16:31:38 +0000112 head = len(self.resultdir) + 1 # remove the / inbetween
113 return path[head:]
mbligh0692e472007-08-30 16:07:53 +0000114
115
mbligh362ab3d2007-08-30 11:24:04 +0000116 def control_get(self):
117 return self.control
118
mblighcaa605c2006-10-02 00:37:35 +0000119
apwde1503a2006-10-10 08:34:21 +0000120 def harness_select(self, which):
121 self.harness = harness.select(which, self)
122
123
apw059e1b12006-10-12 17:18:26 +0000124 def config_set(self, name, value):
125 self.config.set(name, value)
126
127
128 def config_get(self, name):
129 return self.config.get(name)
130
mbligh8baa2ea2006-12-17 23:01:24 +0000131 def setup_dirs(self, results_dir, tmp_dir):
mbligh1e8858e2006-11-24 22:18:35 +0000132 if not tmp_dir:
133 tmp_dir = self.tmpdir + '/build'
134 if not os.path.exists(tmp_dir):
135 os.mkdir(tmp_dir)
136 if not os.path.isdir(tmp_dir):
137 raise "Temp dir (%s) is not a dir - args backwards?" \
138 % self.tmpdir
139
140 # We label the first build "build" and then subsequent ones
141 # as "build.2", "build.3", etc. Whilst this is a little bit
142 # inconsistent, 99.9% of jobs will only have one build
143 # (that's not done as kernbench, sparse, or buildtest),
144 # so it works out much cleaner. One of life's comprimises.
145 if not results_dir:
146 results_dir = os.path.join(self.resultdir, 'build')
147 i = 2
148 while os.path.exists(results_dir):
149 results_dir = os.path.join(self.resultdir, 'build.%d' % i)
mblighd9223fc2006-11-26 17:19:54 +0000150 i += 1
mbligh1e8858e2006-11-24 22:18:35 +0000151 if not os.path.exists(results_dir):
152 os.mkdir(results_dir)
mbligh72b88fc2006-12-16 18:41:35 +0000153
mbligh8baa2ea2006-12-17 23:01:24 +0000154 return (results_dir, tmp_dir)
155
156
157 def xen(self, base_tree, results_dir = '', tmp_dir = '', leave = False, \
158 kjob = None ):
159 """Summon a xen object"""
160 (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
161 build_dir = 'xen'
162 return xen.xen(self, base_tree, results_dir, tmp_dir, build_dir, leave, kjob)
163
164
165 def kernel(self, base_tree, results_dir = '', tmp_dir = '', leave = False):
166 """Summon a kernel object"""
167 (results_dir, tmp_dir) = self.setup_dirs(results_dir, tmp_dir)
168 build_dir = 'linux'
169 return kernel.kernel(self, base_tree, results_dir, tmp_dir, build_dir, leave)
mblighf4c35322006-03-13 01:01:10 +0000170
mblighcaa605c2006-10-02 00:37:35 +0000171
mblighfadca202006-09-23 04:40:01 +0000172 def barrier(self, *args):
173 """Create a barrier object"""
174 return barrier.barrier(*args)
175
mblighcaa605c2006-10-02 00:37:35 +0000176
mbligh4b089662006-06-14 22:34:58 +0000177 def setup_dep(self, deps):
mblighc86b0b42006-07-28 17:35:28 +0000178 """Set up the dependencies for this test.
179
180 deps is a list of libraries required for this test.
181 """
mbligh4b089662006-06-14 22:34:58 +0000182 for dep in deps:
183 try:
184 os.chdir(self.autodir + '/deps/' + dep)
185 system('./' + dep + '.py')
186 except:
187 error = "setting up dependency " + dep + "\n"
mbligh72b88fc2006-12-16 18:41:35 +0000188 raise UnhandledError(error)
mbligh4b089662006-06-14 22:34:58 +0000189
190
mbligh72b88fc2006-12-16 18:41:35 +0000191 def __runtest(self, url, tag, args, dargs):
192 try:
mblighd016ecc2006-11-25 21:41:07 +0000193 test.runtest(self, url, tag, args, dargs)
mbligh72b88fc2006-12-16 18:41:35 +0000194 except AutotestError:
195 raise
196 except:
197 raise UnhandledError('running test ' + \
198 self.__class__.__name__ + "\n")
apwf1a81162006-04-25 10:10:29 +0000199
mblighcaa605c2006-10-02 00:37:35 +0000200
mblighd016ecc2006-11-25 21:41:07 +0000201 def runtest(self, tag, url, *args):
202 raise "Deprecated call to job.runtest. Use run_test instead"
203
204
205 def run_test(self, url, *args, **dargs):
mblighc86b0b42006-07-28 17:35:28 +0000206 """Summon a test object and run it.
207
208 tag
209 tag to add to testname
mbligh12a7df72006-10-06 03:54:33 +0000210 url
211 url of the test to run
mblighc86b0b42006-07-28 17:35:28 +0000212 """
mbligh12a7df72006-10-06 03:54:33 +0000213
mblighd016ecc2006-11-25 21:41:07 +0000214 if not url:
215 raise "Test name is invalid. Switched arguments?"
mbligh12a7df72006-10-06 03:54:33 +0000216 (group, name) = test.testname(url)
mblighd016ecc2006-11-25 21:41:07 +0000217 tag = None
218 if dargs.has_key('tag'):
219 tag = dargs['tag']
220 del dargs['tag']
apw5a7335c2007-03-12 20:32:40 +0000221 if tag:
222 name += '.' + tag
apwf1a81162006-04-25 10:10:29 +0000223 try:
224 try:
mblighd016ecc2006-11-25 21:41:07 +0000225 self.__runtest(url, tag, args, dargs)
apwf1a81162006-04-25 10:10:29 +0000226 except Exception, detail:
227 self.record("FAIL " + name + " " + \
228 detail.__str__() + "\n")
229
230 raise
231 else:
232 self.record("GOOD " + name + \
mblighd9223fc2006-11-26 17:19:54 +0000233 " completed successfully\n")
apwf1a81162006-04-25 10:10:29 +0000234 except TestError:
235 return 0
236 except:
237 raise
238 else:
239 return 1
apw0865f482006-03-30 18:50:19 +0000240
mblighd7fb4a62006-10-01 00:57:53 +0000241
mbligh88ab90f2007-08-29 15:52:49 +0000242 def run_group(self, function, *args):
243 """\
244 function:
245 subroutine to run
246 *args:
247 arguments for the function
248 """
249
250 name = function.__name__
251 # if tag:
252 # name += '.' + tag
253 old_record_prefix = self.record_prefix
254 try:
255 try:
256 self.record("START " + name)
257 self.record_prefix += '\t'
258 function(*args)
259 self.record_prefix = old_record_prefix
260 self.record("END %s GOOD" % name)
261 except:
262 self.record_prefix = old_record_prefix
263 self.record("END %s FAIL" % name)
264 # We don't want to raise up an error higher if it's just
265 # a TestError - we want to carry on to other tests. Hence
266 # this outer try/except block.
267 except TestError:
268 pass
269 except:
270 raise TestError(name + ' failed\n' + format_error())
271
272
mblighc2359852007-08-28 18:11:48 +0000273 def filesystem(self, device, mountpoint = None, loop_size = 0):
mblighd7fb4a62006-10-01 00:57:53 +0000274 if not mountpoint:
275 mountpoint = self.tmpdir
mblighc2359852007-08-28 18:11:48 +0000276 return filesystem.filesystem(self, device, mountpoint,loop_size)
mblighd7fb4a62006-10-01 00:57:53 +0000277
mblighcaa605c2006-10-02 00:37:35 +0000278
279 def reboot(self, tag='autotest'):
apwde1503a2006-10-10 08:34:21 +0000280 self.harness.run_reboot()
mblighcaa605c2006-10-02 00:37:35 +0000281 self.bootloader.boot_once(tag)
282 system("reboot")
apw0778a2f2006-10-06 03:11:40 +0000283 self.quit()
mblighcaa605c2006-10-02 00:37:35 +0000284
285
apw0865f482006-03-30 18:50:19 +0000286 def noop(self, text):
287 print "job: noop: " + text
288
mblighcaa605c2006-10-02 00:37:35 +0000289
apw0865f482006-03-30 18:50:19 +0000290 # Job control primatives.
mblighc86b0b42006-07-28 17:35:28 +0000291
apw8fef4ac2006-10-10 22:53:37 +0000292 def __parallel_execute(self, func, *args):
293 func(*args)
294
295
mblighc86b0b42006-07-28 17:35:28 +0000296 def parallel(self, *tasklist):
297 """Run tasks in parallel"""
apw8fef4ac2006-10-10 22:53:37 +0000298
299 pids = []
300 for task in tasklist:
301 pids.append(fork_start(self.resultdir,
302 lambda: self.__parallel_execute(*task)))
303 for pid in pids:
304 fork_waitfor(self.resultdir, pid)
apw0865f482006-03-30 18:50:19 +0000305
mblighcaa605c2006-10-02 00:37:35 +0000306
apw0865f482006-03-30 18:50:19 +0000307 def quit(self):
mblighc86b0b42006-07-28 17:35:28 +0000308 # XXX: should have a better name.
apwde1503a2006-10-10 08:34:21 +0000309 self.harness.run_pause()
apwf2c66602006-04-27 14:11:25 +0000310 raise JobContinue("more to come")
apw0865f482006-03-30 18:50:19 +0000311
mblighcaa605c2006-10-02 00:37:35 +0000312
apw0865f482006-03-30 18:50:19 +0000313 def complete(self, status):
mblighc86b0b42006-07-28 17:35:28 +0000314 """Clean up and exit"""
apw0865f482006-03-30 18:50:19 +0000315 # We are about to exit 'complete' so clean up the control file.
316 try:
apwecf41b72006-03-31 14:00:55 +0000317 os.unlink(self.control + '.state')
apw0865f482006-03-30 18:50:19 +0000318 except:
319 pass
mbligh61a6c1a2006-12-25 01:26:38 +0000320 self.harness.run_complete()
apw1b021902006-04-03 17:02:56 +0000321 sys.exit(status)
apw0865f482006-03-30 18:50:19 +0000322
mblighcaa605c2006-10-02 00:37:35 +0000323
apw0865f482006-03-30 18:50:19 +0000324 steps = []
325 def next_step(self, step):
mblighc86b0b42006-07-28 17:35:28 +0000326 """Define the next step"""
apw83f8d772006-04-27 14:12:56 +0000327 step[0] = step[0].__name__
apw0865f482006-03-30 18:50:19 +0000328 self.steps.append(step)
apwecf41b72006-03-31 14:00:55 +0000329 pickle.dump(self.steps, open(self.control + '.state', 'w'))
apw0865f482006-03-30 18:50:19 +0000330
mblighcaa605c2006-10-02 00:37:35 +0000331
apw83f8d772006-04-27 14:12:56 +0000332 def step_engine(self):
mblighc86b0b42006-07-28 17:35:28 +0000333 """the stepping engine -- if the control file defines
334 step_init we will be using this engine to drive multiple runs.
335 """
336 """Do the next step"""
apw83f8d772006-04-27 14:12:56 +0000337 lcl = dict({'job': self})
338
339 str = """
340from error import *
341from autotest_utils import *
342"""
343 exec(str, lcl, lcl)
344 execfile(self.control, lcl, lcl)
345
mblighd9223fc2006-11-26 17:19:54 +0000346 state = self.control + '.state'
apw0865f482006-03-30 18:50:19 +0000347 # If there is a mid-job state file load that in and continue
348 # where it indicates. Otherwise start stepping at the passed
349 # entry.
350 try:
mblighd9223fc2006-11-26 17:19:54 +0000351 self.steps = pickle.load(open(state, 'r'))
apw0865f482006-03-30 18:50:19 +0000352 except:
apw83f8d772006-04-27 14:12:56 +0000353 if lcl.has_key('step_init'):
354 self.next_step([lcl['step_init']])
apw0865f482006-03-30 18:50:19 +0000355
356 # Run the step list.
357 while len(self.steps) > 0:
apwfd922bb2006-04-04 07:47:00 +0000358 step = self.steps.pop(0)
mblighd9223fc2006-11-26 17:19:54 +0000359 pickle.dump(self.steps, open(state, 'w'))
apw0865f482006-03-30 18:50:19 +0000360
361 cmd = step.pop(0)
apw83f8d772006-04-27 14:12:56 +0000362 cmd = lcl[cmd]
363 lcl['__cmd'] = cmd
364 lcl['__args'] = step
365 exec("__cmd(*__args)", lcl, lcl)
apw0865f482006-03-30 18:50:19 +0000366
mblighcaa605c2006-10-02 00:37:35 +0000367
apwf1a81162006-04-25 10:10:29 +0000368 def record(self, msg):
mblighc86b0b42006-07-28 17:35:28 +0000369 """Record job-level status"""
apw7db8d0b2006-10-09 08:10:25 +0000370
mblighd9223fc2006-11-26 17:19:54 +0000371 msg = msg.rstrip()
apw7db8d0b2006-10-09 08:10:25 +0000372 # Ensure any continuation lines are marked so we can
373 # detect them in the status file to ensure it is parsable.
mbligh88ab90f2007-08-29 15:52:49 +0000374 msg = re.sub(r"\n", "\n" + self.record_prefix + " ", msg)
375 msg = self.record_prefix + msg
apw7db8d0b2006-10-09 08:10:25 +0000376
apwde1503a2006-10-10 08:34:21 +0000377 self.harness.test_status(msg)
apwf1a81162006-04-25 10:10:29 +0000378 print msg
379 status = self.resultdir + "/status"
mblighd9223fc2006-11-26 17:19:54 +0000380 file(status, "a").write(msg + "\n")
apwce9abe92006-04-27 14:14:04 +0000381
382
mbligh570e93e2006-11-26 05:15:56 +0000383def runjob(control, cont = False, tag = "default", harness_type = ''):
mblighc86b0b42006-07-28 17:35:28 +0000384 """The main interface to this module
385
mbligh72b88fc2006-12-16 18:41:35 +0000386 control
mblighc86b0b42006-07-28 17:35:28 +0000387 The control file to use for this job.
388 cont
389 Whether this is the continuation of a previously started job
390 """
mblighb4eef242007-07-23 18:22:49 +0000391 control = os.path.abspath(control)
apwce9abe92006-04-27 14:14:04 +0000392 state = control + '.state'
393
394 # instantiate the job object ready for the control file.
395 myjob = None
396 try:
397 # Check that the control file is valid
398 if not os.path.exists(control):
399 raise JobError(control + ": control file not found")
400
401 # When continuing, the job is complete when there is no
402 # state file, ensure we don't try and continue.
mblighf3fef462006-09-13 16:05:05 +0000403 if cont and not os.path.exists(state):
apwce9abe92006-04-27 14:14:04 +0000404 sys.exit(1)
mblighf3fef462006-09-13 16:05:05 +0000405 if cont == False and os.path.exists(state):
apwce9abe92006-04-27 14:14:04 +0000406 os.unlink(state)
407
mbligh570e93e2006-11-26 05:15:56 +0000408 myjob = job(control, tag, cont, harness_type)
apwce9abe92006-04-27 14:14:04 +0000409
410 # Load in the users control file, may do any one of:
411 # 1) execute in toto
412 # 2) define steps, and select the first via next_step()
413 myjob.step_engine()
414
apwce9abe92006-04-27 14:14:04 +0000415 except JobContinue:
416 sys.exit(5)
417
418 except JobError, instance:
419 print "JOB ERROR: " + instance.args[0]
420 if myjob != None:
apw510398f2007-03-06 19:19:05 +0000421 myjob.record("ABORT " + instance.args[0] + "\n")
apwce9abe92006-04-27 14:14:04 +0000422 myjob.complete(1)
423
424 except:
mblighfbfb77d2007-02-15 18:54:03 +0000425 if myjob:
426 myjob.harness.run_abort()
apwce9abe92006-04-27 14:14:04 +0000427 # Ensure we cannot continue this job, it is in rictus.
428 if os.path.exists(state):
429 os.unlink(state)
430 raise
mbligh892d37f2007-03-01 17:03:25 +0000431
432 # If we get here, then we assume the job is complete and good.
433 myjob.complete(0)
434