borenet | 11271fe | 2015-07-06 07:43:58 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | |
| 7 | """Run the given command through LLVM's coverage tools.""" |
| 8 | |
| 9 | |
| 10 | import argparse |
| 11 | import json |
| 12 | import os |
borenet | 334e588 | 2015-07-06 11:18:45 -0700 | [diff] [blame] | 13 | import re |
borenet | 11271fe | 2015-07-06 07:43:58 -0700 | [diff] [blame] | 14 | import shlex |
| 15 | import subprocess |
| 16 | import sys |
| 17 | |
| 18 | |
| 19 | BUILDTYPE = 'Coverage' |
| 20 | OUT_DIR = os.path.realpath(os.path.join('out', BUILDTYPE)) |
| 21 | PROFILE_DATA = 'default.profraw' |
| 22 | PROFILE_DATA_MERGED = 'prof_merged' |
| 23 | |
| 24 | |
| 25 | def _fix_filename(filename): |
| 26 | """Return a filename which we can use to identify the file. |
| 27 | |
| 28 | The file paths printed by llvm-cov take the form: |
| 29 | |
| 30 | /path/to/repo/out/dir/../../src/filename.cpp |
| 31 | |
| 32 | And then they're truncated to 22 characters with leading ellipses: |
| 33 | |
| 34 | ...../../src/filename.cpp |
| 35 | |
| 36 | This makes it really tough to determine whether the file actually belongs in |
| 37 | the Skia repo. This function strips out the leading junk so that, if the file |
| 38 | exists in the repo, the returned string matches the end of some relative path |
| 39 | in the repo. This doesn't guarantee correctness, but it's about as close as |
| 40 | we can get. |
| 41 | """ |
| 42 | return filename.split('..')[-1].lstrip('./') |
| 43 | |
| 44 | |
| 45 | def _filter_results(results): |
| 46 | """Filter out any results for files not in the Skia repo. |
| 47 | |
| 48 | We run through the list of checked-in files and determine whether each file |
| 49 | belongs in the repo. Unfortunately, llvm-cov leaves us with fragments of the |
| 50 | file paths, so we can't guarantee accuracy. See the docstring for |
| 51 | _fix_filename for more details. |
| 52 | """ |
| 53 | all_files = subprocess.check_output(['git', 'ls-files']).splitlines() |
| 54 | filtered = [] |
| 55 | for percent, filename in results: |
| 56 | new_file = _fix_filename(filename) |
| 57 | matched = [] |
| 58 | for f in all_files: |
| 59 | if f.endswith(new_file): |
| 60 | matched.append(f) |
| 61 | if len(matched) == 1: |
| 62 | filtered.append((percent, matched[0])) |
| 63 | elif len(matched) > 1: |
| 64 | print >> sys.stderr, ('WARNING: multiple matches for %s; skipping:\n\t%s' |
| 65 | % (new_file, '\n\t'.join(matched))) |
| 66 | print 'Filtered out %d files.' % (len(results) - len(filtered)) |
| 67 | return filtered |
| 68 | |
| 69 | |
| 70 | def run_coverage(cmd): |
| 71 | """Run the given command and return per-file coverage data. |
| 72 | |
| 73 | Assumes that the binary has been built using llvm_coverage_build and that |
| 74 | LLVM 3.6 or newer is installed. |
| 75 | """ |
| 76 | binary_path = os.path.join(OUT_DIR, cmd[0]) |
| 77 | subprocess.call([binary_path] + cmd[1:]) |
| 78 | try: |
| 79 | subprocess.check_call( |
| 80 | ['llvm-profdata', 'merge', PROFILE_DATA, |
| 81 | '-output=%s' % PROFILE_DATA_MERGED]) |
| 82 | finally: |
| 83 | os.remove(PROFILE_DATA) |
| 84 | try: |
| 85 | report = subprocess.check_output( |
| 86 | ['llvm-cov', 'report', '-instr-profile', PROFILE_DATA_MERGED, |
| 87 | binary_path]) |
| 88 | finally: |
| 89 | os.remove(PROFILE_DATA_MERGED) |
| 90 | results = [] |
| 91 | for line in report.splitlines()[2:-2]: |
| 92 | filename, _, _, cover, _, _ = shlex.split(line) |
| 93 | percent = float(cover.split('%')[0]) |
| 94 | results.append((percent, filename)) |
| 95 | results = _filter_results(results) |
| 96 | results.sort() |
| 97 | return results |
| 98 | |
| 99 | |
borenet | 334e588 | 2015-07-06 11:18:45 -0700 | [diff] [blame] | 100 | def _testname(filename): |
| 101 | """Transform the file name into an ingestible test name.""" |
| 102 | return re.sub(r'[^a-zA-Z0-9]', '_', filename) |
| 103 | |
| 104 | |
| 105 | def _nanobench_json(results, properties, key): |
| 106 | """Return the results in JSON format like that produced by nanobench.""" |
| 107 | rv = {} |
| 108 | # Copy over the properties first, then set the 'key' and 'results' keys, |
| 109 | # in order to avoid bad formatting in case the user passes in a properties |
| 110 | # dict containing those keys. |
| 111 | rv.update(properties) |
| 112 | rv['key'] = key |
| 113 | rv['results'] = { |
| 114 | _testname(f): { |
| 115 | 'coverage': { |
| 116 | 'percent': percent, |
| 117 | 'options': { |
| 118 | 'fullname': f, |
| 119 | 'dir': os.path.dirname(f), |
| 120 | }, |
| 121 | }, |
| 122 | } for percent, f in results |
| 123 | } |
| 124 | return rv |
| 125 | |
| 126 | |
| 127 | def _parse_key_value(kv_list): |
| 128 | """Return a dict whose key/value pairs are derived from the given list. |
| 129 | |
| 130 | For example: |
| 131 | |
| 132 | ['k1', 'v1', 'k2', 'v2'] |
| 133 | becomes: |
| 134 | |
| 135 | {'k1': 'v1', |
| 136 | 'k2': 'v2'} |
| 137 | """ |
| 138 | if len(kv_list) % 2 != 0: |
| 139 | raise Exception('Invalid key/value pairs: %s' % kv_list) |
| 140 | |
| 141 | rv = {} |
| 142 | for i in xrange(len(kv_list) / 2): |
| 143 | rv[kv_list[i*2]] = kv_list[i*2+1] |
| 144 | return rv |
| 145 | |
| 146 | |
borenet | 11271fe | 2015-07-06 07:43:58 -0700 | [diff] [blame] | 147 | def main(): |
borenet | 334e588 | 2015-07-06 11:18:45 -0700 | [diff] [blame] | 148 | """Run coverage and generate a report.""" |
| 149 | # Parse args. |
| 150 | parser = argparse.ArgumentParser() |
| 151 | parser.add_argument('--outResultsFile') |
| 152 | parser.add_argument( |
| 153 | '--key', metavar='key_or_value', nargs='+', |
| 154 | help='key/value pairs identifying this bot.') |
| 155 | parser.add_argument( |
| 156 | '--properties', metavar='key_or_value', nargs='+', |
| 157 | help='key/value pairs representing properties of this build.') |
| 158 | args, cmd = parser.parse_known_args() |
| 159 | key = _parse_key_value(args.key) |
| 160 | properties = _parse_key_value(args.properties) |
| 161 | |
| 162 | # Run coverage. |
| 163 | results = run_coverage(cmd) |
| 164 | |
| 165 | # Write results. |
| 166 | format_results = _nanobench_json(results, properties, key) |
| 167 | if args.outResultsFile: |
| 168 | with open(args.outResultsFile, 'w') as f: |
| 169 | json.dump(format_results, f) |
| 170 | else: |
| 171 | print json.dumps(format_results, indent=4, sort_keys=True) |
borenet | 11271fe | 2015-07-06 07:43:58 -0700 | [diff] [blame] | 172 | |
| 173 | |
| 174 | if __name__ == '__main__': |
| 175 | main() |