blob: 2167992bb1befb14f3ff68fe2b49c88bd6ea5d86 [file] [log] [blame]
yaberauneya0d39b832009-11-20 05:45:41 +00001#!/usr/bin/env python
2"""
3 An LTP [execution and] parsing wrapper.
4
5 Used as a second layer for ease-of-use with users as many developers
6 complain about complexity involved with trying to use LTP in my
7 organization -_-.
8
Garrett Cooperfbd580e2012-05-20 14:14:40 -07009 Copyright (C) 2009-2012, Garrett Cooper
yaberauneya0d39b832009-11-20 05:45:41 +000010
11 This program is free software; you can redistribute it and/or modify
12 it under the terms of the GNU General Public License as published by
13 the Free Software Foundation; either version 2 of the License, or
14 (at your option) any later version.
15
16 This program is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU General Public License for more details.
20
21 You should have received a copy of the GNU General Public License along
22 with this program; if not, write to the Free Software Foundation, Inc.,
23 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24"""
25
Garrett Cooperfbd580e2012-05-20 14:14:40 -070026
yaberauneya0d39b832009-11-20 05:45:41 +000027from optparse import OptionGroup, OptionParser
Garrett Cooperfbd580e2012-05-20 14:14:40 -070028import os
29import re
30import sys
31
yaberauneya0d39b832009-11-20 05:45:41 +000032
33class ResultsParseException(Exception):
34 """ Extended class for parsing LTP results. """
35
Garrett Cooperfbd580e2012-05-20 14:14:40 -070036
yaberauneya0d39b832009-11-20 05:45:41 +000037def parse_ltp_results(exec_log, output_log, verbose=0):
Garrett Cooperfbd580e2012-05-20 14:14:40 -070038 """Function for parsing LTP results.
yaberauneya0d39b832009-11-20 05:45:41 +000039
40 1. The exec log is the log with the results in summary form.
41
42 And now a note from our sponsors about exec logs...
43
44 startup='Thu Oct 1 06:42:07 2009'
45 tag=abort01 stime=1254379327 dur=2 exit=exited stat=0 core=no cu=0 cs=16
46 tag=accept01 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=1 cs=0
47 tag=access01 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=0 cs=0
48 tag=access02 stime=1254379329 dur=0 exit=exited stat=0 core=no cu=0 cs=0
49 tag=access03 stime=1254379329 dur=1 exit=exited stat=0 core=no cu=0 cs=1
50
51 [...]
52
53 a. tag is the test tag name.
54 b. stime is the system time at the start of the exec.
55 c. dur is the total duration of the test.
56 d. exit tells you what the result was. Valid values are:
57 - exited
58 - signaled
59 - stopped
60 - unknown
61 See run_child in pan.c.
62 e. stat is the exit status.
63 f. core answers the question: `did I dump core?'.
64 g. cu is the cutime (cumulative user time).
65 h. cs is the cstime (cumulative system time).
66
67 2. The output log is the log with all of the terse results.
68 3. verbose tells us whether or not we need to include the passed results.
69 """
70
71 if not os.access(exec_log, os.R_OK):
72 raise ResultsParseException("Exec log - %s - specified doesn't exist"
73 % exec_log)
74 elif 1 < verbose and not os.access(output_log, os.R_OK):
75 # Need the output log for context to the end user.
76 raise ResultsParseException("Output log - %s - specified doesn't exist"
Garrett Cooperfbd580e2012-05-20 14:14:40 -070077 % output_log)
yaberauneya0d39b832009-11-20 05:45:41 +000078
79 context = None
80
Garrett Cooperfbd580e2012-05-20 14:14:40 -070081 failed = []
yaberauneya0d39b832009-11-20 05:45:41 +000082 passed = 0
83
84 if 2 <= verbose:
Garrett Cooperfbd580e2012-05-20 14:14:40 -070085 passed = []
yaberauneya0d39b832009-11-20 05:45:41 +000086
Garrett Cooperfbd580e2012-05-20 14:14:40 -070087 target_vals = ('exited', '0', 'no')
yaberauneya0d39b832009-11-20 05:45:41 +000088
89 fd = open(exec_log, 'r')
90
91 try:
92 content = fd.read()
93 matches = re.finditer('tag=(?P<tag>\w+).+exit=(?P<exit>\w+) '
94 'stat=(?P<stat>\d+) core=(?P<core>\w+)', content)
95 finally:
Garrett Cooper0ab3aac2011-07-20 10:56:58 -070096 content = None
yaberauneya0d39b832009-11-20 05:45:41 +000097 fd.close()
98
99 if not matches:
100 raise ResultsParseException("No parseable results were found in the "
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700101 "exec log - `%s'." % exec_log)
yaberauneya0d39b832009-11-20 05:45:41 +0000102
103 for match in matches:
104
105 if ((match.group('exit'), match.group('stat'), match.group('core')) !=
106 target_vals):
107 failed.append(match.group('tag'))
108 elif 2 <= verbose:
109 passed.append(match.group('tag'))
110 else:
111 passed += 1
112
113 # Save memory on large files because lists can eat up a fair amount of
114 # memory.
115 matches = None
116
117 if 1 <= verbose:
118
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700119 context = {}
yaberauneya0d39b832009-11-20 05:45:41 +0000120
121 search_tags = failed[:]
122
123 if 2 <= verbose:
124 search_tags += passed
125
126 search_tags.sort()
127
128 fd = open(output_log, 'r')
129
130 try:
131
Garrett Cooper0ab3aac2011-07-20 10:56:58 -0700132 line_iterator = getattr(fd, 'xreadlines', getattr(fd, 'readlines'))
yaberauneya0d39b832009-11-20 05:45:41 +0000133
134 end_output = '<<<execution_status>>>'
135 output_start = '<<<test_output>>>'
136
137 tag_re = re.compile('tag=(\w+)')
138
139 grab_output = False
140
yaberauneya0d39b832009-11-20 05:45:41 +0000141 local_context = ''
142
yaberauneya0d39b832009-11-20 05:45:41 +0000143 search_tag = None
144
yaberauneyabbfbd512009-12-12 07:19:50 +0000145 try:
yaberauneya0d39b832009-11-20 05:45:41 +0000146
yaberauneyabbfbd512009-12-12 07:19:50 +0000147 while True:
yaberauneya0d39b832009-11-20 05:45:41 +0000148
yaberauneyabbfbd512009-12-12 07:19:50 +0000149 line = line_iterator.next()
yaberauneya0d39b832009-11-20 05:45:41 +0000150
yaberauneyabbfbd512009-12-12 07:19:50 +0000151 if line.startswith(end_output):
yaberauneya0d39b832009-11-20 05:45:41 +0000152
yaberauneyabbfbd512009-12-12 07:19:50 +0000153 if search_tag:
154 context[search_tag] = local_context
yaberauneya0d39b832009-11-20 05:45:41 +0000155
yaberauneyabbfbd512009-12-12 07:19:50 +0000156 grab_output = False
157 local_context = ''
158 search_tag = None
yaberauneya0d39b832009-11-20 05:45:41 +0000159
yaberauneyabbfbd512009-12-12 07:19:50 +0000160 if not search_tag:
yaberauneya0d39b832009-11-20 05:45:41 +0000161
yaberauneyabbfbd512009-12-12 07:19:50 +0000162 while True:
yaberauneya0d39b832009-11-20 05:45:41 +0000163
yaberauneyabbfbd512009-12-12 07:19:50 +0000164 line = line_iterator.next()
yaberauneya0d39b832009-11-20 05:45:41 +0000165
yaberauneyabbfbd512009-12-12 07:19:50 +0000166 match = tag_re.match(line)
167
168 if match and match.group(1) in search_tags:
169 search_tag = match.group(1)
170 break
171
172 elif line.startswith(output_start):
173 grab_output = True
174 elif grab_output:
175 local_context += line
176
177 except StopIteration:
178 pass
yaberauneya0d39b832009-11-20 05:45:41 +0000179
180 for k in context.keys():
181 if k not in search_tags:
182 raise ResultsParseException('Leftover token in search '
183 'keys: %s' % k)
184
Garrett Cooper0ab3aac2011-07-20 10:56:58 -0700185 except Exception as exc:
yaberauneya0d39b832009-11-20 05:45:41 +0000186 # XXX (garrcoop): change from Exception to soft error and print
187 # out warning with logging module.
188 raise ResultsParseException('Encountered exception reading output '
189 'for context: %s' % str(exc))
Garrett Cooper0ab3aac2011-07-20 10:56:58 -0700190 finally:
191 fd.close()
yaberauneya0d39b832009-11-20 05:45:41 +0000192
193 return failed, passed, context
194
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700195
yaberauneya0d39b832009-11-20 05:45:41 +0000196def determine_context(output_log, testsuite, test_set, context):
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700197 """Return a set of context values mapping test_set -> context."""
yaberauneya0d39b832009-11-20 05:45:41 +0000198
199 test_set_context = {}
200
201 for test in test_set:
202
203 if test in context:
204 test_context = context[test]
205 del context[test]
206 else:
207 test_context = ('Could not determine context for %s; please see '
208 'output log - %s' % (test, output_log))
209
yaberauneya0d39b832009-11-20 05:45:41 +0000210 test_set_context['%s : %s' % (testsuite, test)] = test_context
211
212 return test_set_context
213
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700214
yaberauneya0d39b832009-11-20 05:45:41 +0000215def print_context(output_dest, header, testsuite_context):
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700216 """Print out testsuite_context to output_dest, heading it up with
217 header.
yaberauneya0d39b832009-11-20 05:45:41 +0000218 """
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700219
yaberauneya0d39b832009-11-20 05:45:41 +0000220 output_dest.write('\n'.join(['', '=' * 40, header, '-' * 40, '']))
221
222 for test, context in testsuite_context.items():
223 output_dest.write('<output test="%s">\n%s\n</output>\n' %
224 (test, context.strip()))
225
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700226
yaberauneya0d39b832009-11-20 05:45:41 +0000227def main():
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700228 """main"""
yaberauneya0d39b832009-11-20 05:45:41 +0000229
230 parser = OptionParser(prog=os.path.basename(sys.argv[0]),
231 usage='usage: %prog [options] test ...',
Garrett Cooper7324c3a2010-11-16 21:48:45 -0800232 version='0.0.2')
yaberauneya0d39b832009-11-20 05:45:41 +0000233
234 ltpdir = os.getenv('LTPROOT', '@prefix@')
235
236 parser.add_option('-l', '--ltp-dir', dest='ltp_dir',
237 default=ltpdir, help='LTP directory [default: %default]')
238 parser.add_option('-L', '--log-dir', dest='log_dir',
239 default=None,
240 help=('directory for [storing and] retrieving logs '
241 '[default: %s/output]' % ltpdir),
242 metavar='DIR')
243 parser.add_option('-p', '--postprocess-only', dest='postprocess_only',
244 default=False, action='store_true',
245 help=("Don't execute runltp; just postprocess logs "
246 "[default: %default]."))
247 parser.add_option('-o', '--output-file', dest='output_file',
248 default=None,
249 help='File to output results')
250 parser.add_option('-r', '--runltp-opts', dest='runltp_opts',
251 default='',
252 help=('options to pass directly to runltp (will '
253 'suppress -q).'))
254
255 group = OptionGroup(parser, 'Logging',
256 'If --summary-mode is 0, then the summary output is '
257 'suppressed. '
258 'If --summary-mode is 1 [the default], then summary '
259 'output will be displayed for test execution'
260 'If --summary-mode is 2, then summary output will be '
261 'provided on a per-test suite basis. If only '
262 'one test suite is specified, this has the same net '
263 "effect as `--summary-mode 1'"
264 'If --verbose is specified once, prints out failed '
265 'test information with additional context. '
266 'If --verbose is specified twice, prints out the '
267 'failed and passed test context, as well as the '
268 'summary.')
269
270 parser.add_option('-s', '--summary-mode', dest='summary_mode', default=1,
271 type='int',
272 help='See Logging.')
273
274 parser.add_option('-v', '--verbose', dest='verbose', default=0,
275 action='count',
276 help=('Increases context verbosity from tests. See '
277 'Verbosity for more details.'))
278 parser.add_option_group(group)
279
280 group = OptionGroup(parser, 'Copyright',
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700281 '%(prog)s version %(version)s, Copyright (C) '
282 '2009-2012, Garrett Cooper %(prog)s comes with '
283 'ABSOLUTELY NO WARRANTY; '
yaberauneya0d39b832009-11-20 05:45:41 +0000284 'This is free software, and you are welcome to '
285 'redistribute it under certain conditions (See the '
286 'license tort in %(file)s for more details).'
287 % { 'file' : os.path.abspath(__file__),
288 'prog' : parser.prog,
289 'version' : parser.version })
290
291 parser.add_option_group(group)
292
293 opts, args = parser.parse_args()
294
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700295 # Remove -q from the opts string, as long as it's a standalone option.
yaberauneya0d39b832009-11-20 05:45:41 +0000296 runltp_opts = re.sub('^((?<!\S)+\-q\s+|\-q|\s+\-q(?!\S))$', '',
297 opts.runltp_opts)
298
299 if not opts.log_dir:
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700300 opts.log_dir = os.path.join(opts.ltp_dir, 'output')
yaberauneya0d39b832009-11-20 05:45:41 +0000301
302 if not opts.summary_mode and not opts.verbose:
303 parser.error('You cannot suppress summary output and disable '
304 'verbosity.')
305 elif opts.summary_mode not in range(3):
306 parser.error('--summary-mode must be a value between 0 and 2.')
307
308 if len(args) == 0:
Garrett Cooper66076082010-11-16 21:49:31 -0800309 # Default to scenarios also used by runltp.
Garrett Cooper7324c3a2010-11-16 21:48:45 -0800310 fd = open(os.path.join(ltpdir, 'scenario-groups/default'), 'r')
311 try:
312 args = fd.readlines()
313 finally:
314 fd.close()
yaberauneya0d39b832009-11-20 05:45:41 +0000315
316 if opts.output_file:
317
318 output_dir = os.path.dirname(opts.output_file)
319
320 if output_dir:
321 # Not cwd; let's check to make sure that the directory does or
322 # does not exist.
323
324 if not os.path.exists(output_dir):
325 # We need to make the directory.
326 os.makedirs(os.path.dirname(opts.output_file))
327 elif not os.path.isdir(os.path.abspath(output_dir)):
328 # Path exists, but isn't a file. Oops!
329 parser.error('Dirname for path specified - %s - is not valid'
330 % output_dir)
331
332 else:
333 # Current path (cwd)
334 opts.output_file = os.path.join(os.getcwd(), opts.output_file)
335
336 output_dest = open(opts.output_file, 'w')
337
338 else:
339
340 output_dest = sys.stdout
341
342 try:
343
344 failed_context = {}
345 passed_context = {}
346
347 failed_count = 0
348 passed_count = 0
349
350 if opts.summary_mode == 2 and len(args) == 1:
351 opts.summary_mode = 1
352
353 for testsuite in args:
354
355 # Iterate over the provided test list
356
357 context = {}
358 exec_log = os.path.join(opts.log_dir, '%s-exec.log' % testsuite)
359 output_log = os.path.join(opts.log_dir, ('%s-output.log'
360 % testsuite))
361
362 failed_subset = {}
363
364 runtest_file = os.path.join(opts.ltp_dir, 'runtest', testsuite)
365
366 if not opts.postprocess_only:
367
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700368 for log in [exec_log, output_log]:
yaberauneya0d39b832009-11-20 05:45:41 +0000369 if os.path.isfile(log):
370 os.remove(log)
371
372 if not os.access(runtest_file, os.R_OK):
373 output_dest.write("%s doesn't exist; skipping "
374 "test\n" % runtest_file)
375 continue
376
Garrett Cooperfbd580e2012-05-20 14:14:40 -0700377 os.system(' '.join([os.path.join(opts.ltp_dir, 'runltp'),
378 runltp_opts, '-f', testsuite,
379 '-l', exec_log, '-o', output_log]))
yaberauneya0d39b832009-11-20 05:45:41 +0000380
381 try:
382
383 failed_subset, passed_css, context = \
384 parse_ltp_results(exec_log, output_log,
385 verbose=opts.verbose)
386
Garrett Cooper0ab3aac2011-07-20 10:56:58 -0700387 except ResultsParseException as rpe:
yaberauneya0d39b832009-11-20 05:45:41 +0000388 output_dest.write('Error encountered when parsing results for '
389 'test - %s: %s\n' % (testsuite, str(rpe)))
390 continue
391
392 failed_count += len(failed_subset)
393
394 failed_subset_context = {}
395 passed_subset_context = {}
396
397 if opts.verbose:
398 failed_subset_context = determine_context(output_log,
399 testsuite,
400 failed_subset,
401 context)
402 if type(passed_css) == list:
403
404 passed_count += len(passed_css)
405
406 if opts.verbose == 2:
407 passed_subset_context = determine_context(output_log,
408 testsuite,
409 passed_css,
410 context)
411
412 else:
413
414 passed_count += passed_css
415
416 if opts.summary_mode == 1:
417
418 failed_context.update(failed_subset_context)
419 passed_context.update(passed_subset_context)
420
421 else:
422
423 if 1 <= opts.verbose:
424 # Print out failed testcases.
425 print_context(output_dest,
426 'FAILED TESTCASES for %s' % testsuite,
427 failed_subset_context)
428
429 if opts.verbose == 2:
430 # Print out passed testcases with context.
431 print_context(output_dest,
432 'PASSED TESTCASES for %s' % testsuite,
433 passed_subset_context)
434
435 if opts.summary_mode == 2:
436 output_dest.write("""
437========================================
438SUMMARY for: %s
439----------------------------------------
440PASS - %d
441FAIL - %d
442----------------------------------------
443""" % (testsuite, passed_count, len(failed_subset)))
444
445 if opts.summary_mode == 1:
446
447 # Print out overall results.
448
449 if 1 <= opts.verbose:
450 # Print out failed testcases with context.
451 print_context(output_dest, "FAILED TESTCASES", failed_context)
452
453 if opts.verbose == 2:
454 # Print out passed testcases with context.
455 print_context(output_dest, "PASSED TESTCASES", passed_context)
456
457 output_dest.write("""
458========================================
459SUMMARY for tests:
460%s
461----------------------------------------
462PASS - %d
463FAIL - %d
464----------------------------------------
465""" % (' '.join(args), passed_count, failed_count))
466
467 finally:
468
469 if output_dest != sys.stdout:
470
471 output_dest.close()
472
473if __name__ == '__main__':
474 main()