Mike Frysinger | d03e6b5 | 2019-08-03 12:49:01 -0400 | [diff] [blame] | 1 | #!/usr/bin/python2 |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 2 | # (C) Copyright IBM Corp. 2006 |
| 3 | # Author: Dustin Kirkland <dustin.kirkland@us.ibm.com> |
| 4 | # Description: |
| 5 | # Input: Two or more files containing results from different executions of |
| 6 | # the LTP. The input can either be file names or the url location |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 7 | # of the ltp.results file. |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 8 | # Output: A report on the following: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 9 | # - The total number of tests executed in each run |
| 10 | # - The testname, sequence number, and output of each run |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 11 | # where the results of those runs differ |
| 12 | # Return: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 13 | # 0 if all runs had identical results |
| 14 | # Non-zero if results differ, or bad input |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 15 | |
| 16 | |
mbligh | e829ba5 | 2008-06-03 15:04:08 +0000 | [diff] [blame] | 17 | import sys, string, re |
mbligh | 831b661 | 2009-05-01 23:12:29 +0000 | [diff] [blame] | 18 | from autotest_lib.client.common_lib import utils |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 19 | |
| 20 | def usage(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 21 | print "\nUsage: \n\ |
| 22 | ltp-diff results1 results2 ... locationN \n\ |
| 23 | Note: location[1,2,N] may be local files or URLs of LTP results\n" |
| 24 | sys.exit(1) |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 25 | |
| 26 | def get_results(results_files): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 27 | """ |
| 28 | Download the results if needed. |
| 29 | Return results of each run in a numerically-indexed dictionary |
| 30 | of dictionaries keyed on testnames. |
| 31 | Return dictionary keyed on unique testnames across all runs. |
| 32 | """ |
| 33 | r = re.compile('(\S+\s+\S+)\s+(\S+)\s+:') |
| 34 | i = 0 |
| 35 | runs = {} |
| 36 | testnames = {} |
| 37 | for file in results_files: |
| 38 | runs[i] = {} |
| 39 | try: |
| 40 | fh = utils.urlopen(file) |
| 41 | results = fh.readlines() |
| 42 | fh.close() |
| 43 | except: |
| 44 | print "ERROR: reading results resource [%s]" % (file) |
| 45 | usage() |
| 46 | for line in results: |
| 47 | try: |
| 48 | s = r.match(line) |
| 49 | testname = s.group(1) |
| 50 | status = s.group(2) |
| 51 | runs[i][testname] = status |
| 52 | testnames[testname] = 1 |
| 53 | except: |
| 54 | pass |
| 55 | i += 1 |
| 56 | return (runs, testnames) |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 57 | |
| 58 | |
| 59 | |
| 60 | def compare_results(runs): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 61 | """ |
| 62 | Loop through all testnames alpahbetically. |
| 63 | Print any testnames with differing results across runs. |
| 64 | Return 1 if any test results across runs differ. |
| 65 | Return 0 if all test results match. |
| 66 | """ |
| 67 | rc = 0 |
| 68 | print "LTP Test Results to Compare" |
| 69 | for i in range(len(runs)): |
| 70 | print " Run[%d]: %d" % (i, len(runs[i].keys())) |
| 71 | print "" |
| 72 | header = 0 |
| 73 | all_testnames = testnames.keys() |
| 74 | all_testnames.sort() |
| 75 | for testname in all_testnames: |
| 76 | differ = 0 |
| 77 | for i in range(1,len(runs)): |
| 78 | # Must handle testcases that executed in one run |
| 79 | # but not another by setting status to "null" |
| 80 | if not runs[i].has_key(testname): |
| 81 | runs[i][testname] = "null" |
| 82 | if not runs[i-1].has_key(testname): |
| 83 | runs[i-1][testname] = "null" |
| 84 | # Check for the results inconsistencies |
| 85 | if runs[i][testname] != runs[i-1][testname]: |
| 86 | differ = 1 |
| 87 | if differ: |
| 88 | if header == 0: |
| 89 | # Print the differences header only once |
| 90 | print "Tests with Inconsistent Results across Runs" |
| 91 | print " %-35s:\t%s" % ("Testname,Sequence", "Run Results") |
| 92 | header = 1 |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 93 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 94 | # Print info if results differ |
| 95 | rc = 1 |
| 96 | testname_cleaned = re.sub('\s+', ',', testname) |
| 97 | print " %-35s:\t" % (testname_cleaned), |
| 98 | all_results = "" |
| 99 | for i in range(len(runs)): |
| 100 | all_results += runs[i][testname] |
| 101 | if i+1<len(runs): |
| 102 | all_results += "/" |
| 103 | print all_results |
| 104 | if rc == 0: |
| 105 | print "All LTP results are identical" |
| 106 | return rc |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 107 | |
| 108 | |
| 109 | ######## |
| 110 | # Main # |
| 111 | ######## |
| 112 | sys.argv.pop(0) |
| 113 | if (len(sys.argv) < 2): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 114 | usage() |
mbligh | c369219 | 2006-09-22 18:08:54 +0000 | [diff] [blame] | 115 | (runs, testnames) = get_results(sys.argv) |
| 116 | rc = compare_results(runs) |
| 117 | sys.exit(rc) |