| Bill Wendling | f4594a3 | 2012-04-20 20:31:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| Bill Wendling | aec4011 | 2011-10-23 20:13:14 +0000 | [diff] [blame] | 2 | import re, string, sys, os, time |
| 3 | |
| 4 | DEBUG = 0 |
| 5 | testDirName = 'llvm-test' |
| 6 | test = ['compile', 'llc', 'jit', 'cbe'] |
| 7 | exectime = ['llc-time', 'jit-time', 'cbe-time',] |
| 8 | comptime = ['llc', 'jit-comptime', 'compile'] |
| 9 | |
| 10 | (tp, exp) = ('compileTime_', 'executeTime_') |
| 11 | |
| 12 | def parse(file): |
| 13 | f=open(file, 'r') |
| 14 | d = f.read() |
| 15 | |
| 16 | #Cleanup weird stuff |
| 17 | d = re.sub(r',\d+:\d','', d) |
| 18 | |
| 19 | r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) |
| 20 | |
| 21 | test = {} |
| 22 | fname = '' |
| 23 | for t in r: |
| 24 | if DEBUG: |
| 25 | print t |
| 26 | if t[0] == 'PASS' or t[0] == 'FAIL' : |
| 27 | tmp = t[2].split(testDirName) |
| 28 | |
| 29 | if DEBUG: |
| 30 | print tmp |
| 31 | |
| 32 | if len(tmp) == 2: |
| 33 | fname = tmp[1].strip('\r\n') |
| 34 | else: |
| 35 | fname = tmp[0].strip('\r\n') |
| 36 | |
| 37 | if not test.has_key(fname) : |
| 38 | test[fname] = {} |
| 39 | |
| 40 | for k in test: |
| 41 | test[fname][k] = 'NA' |
| 42 | test[fname][t[1]] = t[0] |
| 43 | if DEBUG: |
| 44 | print test[fname][t[1]] |
| 45 | else : |
| 46 | try: |
| 47 | n = t[0].split('RESULT-')[1] |
| 48 | |
| 49 | if DEBUG: |
| 50 | print n; |
| 51 | |
| 52 | if n == 'llc' or n == 'jit-comptime' or n == 'compile': |
| 53 | test[fname][tp + n] = float(t[2].split(' ')[2]) |
| 54 | if DEBUG: |
| 55 | print test[fname][tp + n] |
| 56 | |
| 57 | elif n.endswith('-time') : |
| 58 | test[fname][exp + n] = float(t[2].strip('\r\n')) |
| 59 | if DEBUG: |
| 60 | print test[fname][exp + n] |
| 61 | |
| 62 | else : |
| 63 | print "ERROR!" |
| 64 | sys.exit(1) |
| 65 | |
| 66 | except: |
| 67 | continue |
| 68 | |
| 69 | return test |
| 70 | |
| 71 | # Diff results and look for regressions. |
| 72 | def diffResults(d_old, d_new): |
| 73 | |
| 74 | for t in sorted(d_old.keys()) : |
| 75 | if DEBUG: |
| 76 | print t |
| 77 | |
| 78 | if d_new.has_key(t) : |
| 79 | |
| 80 | # Check if the test passed or failed. |
| 81 | for x in test: |
| 82 | if d_old[t].has_key(x): |
| 83 | if d_new[t].has_key(x): |
| 84 | if d_old[t][x] == 'PASS': |
| 85 | if d_new[t][x] != 'PASS': |
| 86 | print t + " *** REGRESSION (" + x + ")\n" |
| 87 | else: |
| 88 | if d_new[t][x] == 'PASS': |
| 89 | print t + " * NEW PASS (" + x + ")\n" |
| 90 | |
| 91 | else : |
| 92 | print t + "*** REGRESSION (" + x + ")\n" |
| 93 | |
| 94 | # For execution time, if there is no result, its a fail. |
| 95 | for x in exectime: |
| 96 | if d_old[t].has_key(tp + x): |
| 97 | if not d_new[t].has_key(tp + x): |
| 98 | print t + " *** REGRESSION (" + tp + x + ")\n" |
| 99 | |
| 100 | else : |
| 101 | if d_new[t].has_key(tp + x): |
| 102 | print t + " * NEW PASS (" + tp + x + ")\n" |
| 103 | |
| 104 | |
| 105 | for x in comptime: |
| 106 | if d_old[t].has_key(exp + x): |
| 107 | if not d_new[t].has_key(exp + x): |
| 108 | print t + " *** REGRESSION (" + exp + x + ")\n" |
| 109 | |
| 110 | else : |
| 111 | if d_new[t].has_key(exp + x): |
| 112 | print t + " * NEW PASS (" + exp + x + ")\n" |
| 113 | |
| 114 | else : |
| 115 | print t + ": Removed from test-suite.\n" |
| 116 | |
| 117 | |
| 118 | #Main |
| 119 | if len(sys.argv) < 3 : |
| 120 | print 'Usage:', sys.argv[0], \ |
| 121 | '<old log> <new log>' |
| 122 | sys.exit(-1) |
| 123 | |
| 124 | d_old = parse(sys.argv[1]) |
| 125 | d_new = parse(sys.argv[2]) |
| 126 | |
| 127 | |
| 128 | diffResults(d_old, d_new) |
| 129 | |
| 130 | |