Josh Haberman | 67c727c | 2016-03-04 14:21:18 -0800 | [diff] [blame] | 1 | """Gathers output from test runs and create an XML file in JUnit format. |
Josh Haberman | 38bc155 | 2016-02-23 14:28:02 -0800 | [diff] [blame] | 2 | |
| 3 | The output files from the individual tests have been written in a directory |
| 4 | structure like: |
| 5 | |
Josh Haberman | 67c727c | 2016-03-04 14:21:18 -0800 | [diff] [blame] | 6 | $DIR/joblog (output from "parallel --joblog joblog") |
Josh Haberman | 38bc155 | 2016-02-23 14:28:02 -0800 | [diff] [blame] | 7 | $DIR/logs/1/cpp/stdout |
| 8 | $DIR/logs/1/cpp/stderr |
| 9 | $DIR/logs/1/csharp/stdout |
| 10 | $DIR/logs/1/csharp/stderr |
| 11 | $DIR/logs/1/java_jdk7/stdout |
| 12 | $DIR/logs/1/java_jdk7/stderr |
| 13 | etc. |
| 14 | |
| 15 | This script bundles them into a single output XML file so Jenkins can show |
Josh Haberman | 67c727c | 2016-03-04 14:21:18 -0800 | [diff] [blame] | 16 | detailed test results. It runs as the last step before the Jenkins build |
| 17 | finishes. |
Josh Haberman | 38bc155 | 2016-02-23 14:28:02 -0800 | [diff] [blame] | 18 | """ |
| 19 | |
| 20 | import os; |
| 21 | import sys; |
| 22 | from yattag import Doc |
| 23 | from collections import defaultdict |
| 24 | |
| 25 | def readtests(basedir): |
| 26 | tests = defaultdict(dict) |
| 27 | |
| 28 | # Sample input (note: separators are tabs). |
| 29 | # |
| 30 | # Seq Host Starttime Runtime Send Receive Exitval Signal Command |
| 31 | # 1 : 1456263838.313 0.005 0 0 0 0 echo A |
| 32 | with open(basedir + "/joblog") as jobs: |
| 33 | firstline = next(jobs) |
| 34 | for line in jobs: |
| 35 | values = line.split("\t") |
| 36 | |
| 37 | name = values[8].split()[-1] |
| 38 | test = tests[name] |
| 39 | test["name"] = name |
| 40 | test["time"] = values[3] |
| 41 | |
| 42 | exitval = values[6] |
| 43 | if int(exitval): |
| 44 | # We don't have a more specific message. User should look at stderr. |
| 45 | test["failure"] = "TEST FAILURE" |
| 46 | else: |
| 47 | test["failure"] = False |
| 48 | |
| 49 | for testname in os.listdir(basedir + "/logs/1"): |
| 50 | test = tests[testname] |
| 51 | |
| 52 | with open(basedir + "/logs/1/" + testname + "/stdout") as f: |
| 53 | test["stdout"] = f.read() |
| 54 | |
| 55 | with open(basedir + "/logs/1/" + testname + "/stderr") as f: |
| 56 | test["stderr"] = f.read() |
| 57 | |
| 58 | # The cpp test is special since it doesn't run under parallel so doesn't show |
| 59 | # up in the job log. |
| 60 | tests["cpp"]["name"] = "cpp" |
| 61 | |
Josh Haberman | 2bda98f | 2016-03-03 17:04:36 -0800 | [diff] [blame] | 62 | with open(basedir + '/logs/1/cpp/build_time', 'r') as f: |
| 63 | tests["cpp"]["time"] = f.read().strip() |
Josh Haberman | 38bc155 | 2016-02-23 14:28:02 -0800 | [diff] [blame] | 64 | tests["cpp"]["failure"] = False |
| 65 | |
| 66 | ret = tests.values() |
| 67 | ret.sort(key=lambda x: x["name"]) |
| 68 | |
| 69 | return ret |
| 70 | |
| 71 | def genxml(tests): |
| 72 | doc, tag, text = Doc().tagtext() |
| 73 | |
| 74 | with tag("testsuites"): |
| 75 | with tag("testsuite", name="Protobuf Tests"): |
| 76 | for test in tests: |
| 77 | with tag("testcase", name=test["name"], classname=test["name"], |
| 78 | time=test["time"]): |
| 79 | with tag("system-out"): |
| 80 | text(test["stdout"]) |
| 81 | with tag("system-err"): |
| 82 | text(test["stderr"]) |
| 83 | if test["failure"]: |
| 84 | with tag("failure"): |
| 85 | text(test["failure"]) |
| 86 | |
| 87 | return doc.getvalue() |
| 88 | |
| 89 | sys.stderr.write("make_test_output.py: writing XML from directory: " + |
| 90 | sys.argv[1] + "\n"); |
| 91 | print genxml(readtests(sys.argv[1])) |