Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
| 2 | """Generate summary report for ChromeOS toolchain waterfalls.""" |
| 3 | |
| 4 | # Desired future features (to be added): |
| 5 | # - arguments to allow generating only the main waterfall report, |
| 6 | # or only the rotating builder reports, or only the failures |
| 7 | # report; or the waterfall reports without the failures report. |
| 8 | # - Better way of figuring out which dates/builds to generate |
| 9 | # reports for: probably an argument specifying a date or a date |
| 10 | # range, then use something like the new buildbot utils to |
| 11 | # query the build logs to find the right build numbers for the |
| 12 | # builders for the specified dates. |
| 13 | # - Store/get the json/data files in mobiletc-prebuild's x20 area. |
| 14 | # - Update data in json file to reflect, for each testsuite, which |
| 15 | # tests are not expected to run on which boards; update this |
| 16 | # script to use that data appropriately. |
| 17 | # - Make sure user's prodaccess is up-to-date before trying to use |
| 18 | # this script. |
| 19 | # - Add some nice formatting/highlighting to reports. |
| 20 | |
| 21 | from __future__ import print_function |
| 22 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 23 | import argparse |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 24 | import getpass |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 25 | import json |
| 26 | import os |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 27 | import re |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 28 | import shutil |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 29 | import sys |
| 30 | import time |
| 31 | |
| 32 | from cros_utils import command_executer |
| 33 | |
| 34 | # All the test suites whose data we might want for the reports. |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 35 | TESTS = (('bvt-inline', 'HWTest'), ('bvt-cq', 'HWTest'), ('security', 'HWTest'), |
Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 36 | ('kernel_daily_regression', 'HWTest'), ('kernel_daily_benchmarks', |
| 37 | 'HWTest'),) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 38 | |
| 39 | # The main waterfall builders, IN THE ORDER IN WHICH WE WANT THEM |
| 40 | # LISTED IN THE REPORT. |
| 41 | WATERFALL_BUILDERS = [ |
| 42 | 'amd64-gcc-toolchain', 'arm-gcc-toolchain', 'arm64-gcc-toolchain', |
| 43 | 'x86-gcc-toolchain', 'amd64-llvm-toolchain', 'arm-llvm-toolchain', |
| 44 | 'arm64-llvm-toolchain', 'x86-llvm-toolchain', 'amd64-llvm-next-toolchain', |
| 45 | 'arm-llvm-next-toolchain', 'arm64-llvm-next-toolchain', |
| 46 | 'x86-llvm-next-toolchain' |
| 47 | ] |
| 48 | |
Manoj Gupta | 9c0b33b | 2016-12-15 14:52:25 -0800 | [diff] [blame] | 49 | DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/' |
| 50 | ARCHIVE_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-reports/' |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 51 | DOWNLOAD_DIR = '/tmp/waterfall-logs' |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 52 | MAX_SAVE_RECORDS = 7 |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 53 | BUILD_DATA_FILE = '%s/build-data.txt' % DATA_DIR |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 54 | GCC_ROTATING_BUILDER = 'gcc_toolchain' |
| 55 | LLVM_ROTATING_BUILDER = 'llvm_next_toolchain' |
| 56 | ROTATING_BUILDERS = [GCC_ROTATING_BUILDER, LLVM_ROTATING_BUILDER] |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 57 | |
| 58 | # For int-to-string date conversion. Note, the index of the month in this |
| 59 | # list needs to correspond to the month's integer value. i.e. 'Sep' must |
| 60 | # be as MONTHS[9]. |
| 61 | MONTHS = [ |
| 62 | '', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', |
| 63 | 'Nov', 'Dec' |
| 64 | ] |
| 65 | |
| 66 | |
| 67 | def format_date(int_date): |
| 68 | """Convert an integer date to a string date. YYYYMMDD -> YYYY-MMM-DD""" |
| 69 | |
| 70 | if int_date == 0: |
| 71 | return 'today' |
| 72 | |
| 73 | tmp_date = int_date |
| 74 | day = tmp_date % 100 |
| 75 | tmp_date = tmp_date / 100 |
| 76 | month = tmp_date % 100 |
| 77 | year = tmp_date / 100 |
| 78 | |
| 79 | month_str = MONTHS[month] |
| 80 | date_str = '%d-%s-%d' % (year, month_str, day) |
| 81 | return date_str |
| 82 | |
| 83 | |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 84 | def EmailReport(report_file, report_type, date): |
| 85 | subject = '%s Waterfall Summary report, %s' % (report_type, date) |
| 86 | email_to = getpass.getuser() |
Rahul Chaudhry | 213f3c0 | 2016-12-06 10:47:05 -0800 | [diff] [blame] | 87 | sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr' |
| 88 | command = ('%s --to=%s@google.com --subject="%s" --body_file=%s' % |
| 89 | (sendgmr_path, email_to, subject, report_file)) |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 90 | command_executer.GetCommandExecuter().RunCommand(command) |
| 91 | |
| 92 | |
| 93 | def PruneOldFailures(failure_dict, int_date): |
| 94 | earliest_date = int_date - MAX_SAVE_RECORDS |
| 95 | for suite in failure_dict: |
| 96 | suite_dict = failure_dict[suite] |
| 97 | test_keys_to_remove = [] |
| 98 | for test in suite_dict: |
| 99 | test_dict = suite_dict[test] |
| 100 | msg_keys_to_remove = [] |
| 101 | for msg in test_dict: |
| 102 | fails = test_dict[msg] |
| 103 | i = 0 |
| 104 | while i < len(fails) and fails[i][0] <= earliest_date: |
| 105 | i += 1 |
| 106 | new_fails = fails[i:] |
| 107 | test_dict[msg] = new_fails |
| 108 | if len(new_fails) == 0: |
| 109 | msg_keys_to_remove.append(msg) |
| 110 | |
| 111 | for k in msg_keys_to_remove: |
| 112 | del test_dict[k] |
| 113 | |
| 114 | suite_dict[test] = test_dict |
| 115 | if len(test_dict) == 0: |
| 116 | test_keys_to_remove.append(test) |
| 117 | |
| 118 | for k in test_keys_to_remove: |
| 119 | del suite_dict[k] |
| 120 | |
| 121 | failure_dict[suite] = suite_dict |
| 122 | |
| 123 | |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 124 | def GetBuildID(build_bot, date): |
| 125 | """Get the build id for a build_bot at a given date.""" |
Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 126 | day = '{day:02d}'.format(day=date % 100) |
| 127 | mon = MONTHS[date / 100 % 100] |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 128 | date_string = mon + ' ' + day |
| 129 | if build_bot in WATERFALL_BUILDERS: |
| 130 | url = 'https://uberchromegw.corp.google.com/i/chromeos/' + \ |
| 131 | 'builders/%s?numbuilds=200' % build_bot |
| 132 | if build_bot in ROTATING_BUILDERS: |
| 133 | url = 'https://uberchromegw.corp.google.com/i/chromiumos.tryserver/' + \ |
| 134 | 'builders/%s?numbuilds=200' % build_bot |
Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 135 | command = 'sso_client %s' % url |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 136 | retval = 1 |
| 137 | retry_time = 3 |
| 138 | while retval and retry_time: |
| 139 | retval, output, _ = \ |
| 140 | command_executer.GetCommandExecuter().RunCommandWOutput(command, \ |
| 141 | print_to_console=False) |
| 142 | retry_time -= 1 |
| 143 | |
| 144 | if retval: |
| 145 | return [] |
| 146 | |
| 147 | out = output.split('\n') |
| 148 | line_num = 0 |
| 149 | build_id = [] |
| 150 | # Parse the output like this |
| 151 | # <td>Dec 14 10:55</td> |
| 152 | # <td class="revision">??</td> |
| 153 | # <td failure</td><td><a href="../builders/gcc_toolchain/builds/109">#109</a> |
| 154 | while line_num < len(out): |
| 155 | if date_string in out[line_num]: |
| 156 | if line_num + 2 < len(out): |
| 157 | build_num_line = out[line_num + 2] |
| 158 | raw_num = re.findall(r'builds/\d+', build_num_line) |
| 159 | # raw_num is ['builds/109'] in the example. |
| 160 | if raw_num: |
| 161 | build_id.append(int(raw_num[0].split('/')[1])) |
| 162 | line_num += 1 |
| 163 | return build_id |
| 164 | |
| 165 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 166 | def GenerateFailuresReport(fail_dict, date): |
| 167 | filename = 'waterfall_report.failures.%s.txt' % date |
| 168 | date_string = format_date(date) |
| 169 | with open(filename, 'w') as out_file: |
| 170 | # Write failure report section. |
| 171 | out_file.write('\n\nSummary of Test Failures as of %s\n\n' % date_string) |
| 172 | |
| 173 | # We want to sort the errors and output them in order of the ones that occur |
| 174 | # most often. So we have to collect the data about all of them, then sort |
| 175 | # it. |
| 176 | error_groups = [] |
| 177 | for suite in fail_dict: |
| 178 | suite_dict = fail_dict[suite] |
| 179 | if suite_dict: |
| 180 | for test in suite_dict: |
| 181 | test_dict = suite_dict[test] |
| 182 | for err_msg in test_dict: |
| 183 | err_list = test_dict[err_msg] |
| 184 | sorted_list = sorted(err_list, key=lambda x: x[0], reverse=True) |
| 185 | err_group = [len(sorted_list), suite, test, err_msg, sorted_list] |
| 186 | error_groups.append(err_group) |
| 187 | |
| 188 | # Sort the errors by the number of errors of each type. Then output them in |
| 189 | # order. |
| 190 | sorted_errors = sorted(error_groups, key=lambda x: x[0], reverse=True) |
| 191 | for i in range(0, len(sorted_errors)): |
| 192 | err_group = sorted_errors[i] |
| 193 | suite = err_group[1] |
| 194 | test = err_group[2] |
| 195 | err_msg = err_group[3] |
| 196 | err_list = err_group[4] |
| 197 | out_file.write('Suite: %s\n' % suite) |
| 198 | out_file.write(' %s (%d failures)\n' % (test, len(err_list))) |
| 199 | out_file.write(' (%s)\n' % err_msg) |
| 200 | for i in range(0, len(err_list)): |
| 201 | err = err_list[i] |
| 202 | out_file.write(' %s, %s, %s\n' % (format_date(err[0]), err[1], |
| 203 | err[2])) |
| 204 | out_file.write('\n') |
| 205 | |
| 206 | print('Report generated in %s.' % filename) |
| 207 | return filename |
| 208 | |
| 209 | |
| 210 | def GenerateWaterfallReport(report_dict, fail_dict, waterfall_type, date, |
| 211 | omit_failures): |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 212 | """Write out the actual formatted report.""" |
| 213 | |
| 214 | filename = 'waterfall_report.%s_waterfall.%s.txt' % (waterfall_type, date) |
| 215 | |
| 216 | date_string = '' |
| 217 | date_list = report_dict['date'] |
| 218 | num_dates = len(date_list) |
| 219 | i = 0 |
| 220 | for d in date_list: |
| 221 | date_string += d |
| 222 | if i < num_dates - 1: |
| 223 | date_string += ', ' |
| 224 | i += 1 |
| 225 | |
| 226 | if waterfall_type == 'main': |
| 227 | report_list = WATERFALL_BUILDERS |
| 228 | else: |
| 229 | report_list = report_dict.keys() |
| 230 | |
| 231 | with open(filename, 'w') as out_file: |
| 232 | # Write Report Header |
| 233 | out_file.write('\nStatus of %s Waterfall Builds from %s\n\n' % |
| 234 | (waterfall_type, date_string)) |
| 235 | out_file.write(' ' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 236 | ' kernel kernel\n') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 237 | out_file.write(' Build bvt- bvt-cq ' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 238 | ' security daily daily\n') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 239 | out_file.write(' status inline ' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 240 | ' regression benchmarks\n') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 241 | out_file.write(' [P/ F/ DR]* [P/ F /DR]* ' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 242 | '[P/ F/ DR]* [P/ F/ DR]* [P/ F/ DR]*\n\n') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 243 | |
| 244 | # Write daily waterfall status section. |
| 245 | for i in range(0, len(report_list)): |
| 246 | builder = report_list[i] |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 247 | if builder == 'date': |
| 248 | continue |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 249 | |
| 250 | if builder not in report_dict: |
| 251 | out_file.write('Unable to find information for %s.\n\n' % builder) |
| 252 | continue |
| 253 | |
| 254 | build_dict = report_dict[builder] |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 255 | status = build_dict.get('build_status', 'bad') |
| 256 | inline = build_dict.get('bvt-inline', '[??/ ?? /??]') |
| 257 | cq = build_dict.get('bvt-cq', '[??/ ?? /??]') |
| 258 | inline_color = build_dict.get('bvt-inline-color', '') |
| 259 | cq_color = build_dict.get('bvt-cq-color', '') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 260 | if 'x86' not in builder: |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 261 | security = build_dict.get('security', '[??/ ?? /??]') |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 262 | security_color = build_dict.get('security-color', '') |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 263 | if 'gcc' in builder: |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 264 | regression = build_dict.get('kernel_daily_regression', '[??/ ?? /??]') |
| 265 | bench = build_dict.get('kernel_daily_benchmarks', '[??/ ?? /??]') |
| 266 | regression_color = build_dict.get('kernel_daily_regression-color', '') |
| 267 | bench_color = build_dict.get('kernel_daily_benchmarks-color', '') |
| 268 | out_file.write(' %6s %6s' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 269 | ' %6s %6s %6s\n' % |
| 270 | (inline_color, cq_color, security_color, |
| 271 | regression_color, bench_color)) |
| 272 | out_file.write('%25s %3s %s %s %s %s %s\n' % |
| 273 | (builder, status, inline, cq, security, regression, |
| 274 | bench)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 275 | else: |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 276 | out_file.write(' %6s %6s' |
Luis A. Lozano | 4c5de98 | 2017-05-12 18:02:06 -0700 | [diff] [blame] | 277 | ' %6s\n' % (inline_color, cq_color, |
| 278 | security_color)) |
| 279 | out_file.write('%25s %3s %s %s %s\n' % (builder, status, inline, cq, |
| 280 | security)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 281 | else: |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 282 | out_file.write(' %6s %6s\n' % |
| 283 | (inline_color, cq_color)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 284 | out_file.write('%25s %3s %s %s\n' % (builder, status, inline, cq)) |
| 285 | if 'build_link' in build_dict: |
| 286 | out_file.write('%s\n\n' % build_dict['build_link']) |
| 287 | |
| 288 | out_file.write('\n\n*P = Number of tests in suite that Passed; F = ' |
| 289 | 'Number of tests in suite that Failed; DR = Number of tests' |
| 290 | ' in suite that Didn\'t Run.\n') |
| 291 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 292 | if omit_failures: |
| 293 | print('Report generated in %s.' % filename) |
| 294 | return filename |
| 295 | |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 296 | # Write failure report section. |
| 297 | out_file.write('\n\nSummary of Test Failures as of %s\n\n' % date_string) |
| 298 | |
| 299 | # We want to sort the errors and output them in order of the ones that occur |
| 300 | # most often. So we have to collect the data about all of them, then sort |
| 301 | # it. |
| 302 | error_groups = [] |
| 303 | for suite in fail_dict: |
| 304 | suite_dict = fail_dict[suite] |
| 305 | if suite_dict: |
| 306 | for test in suite_dict: |
| 307 | test_dict = suite_dict[test] |
| 308 | for err_msg in test_dict: |
| 309 | err_list = test_dict[err_msg] |
| 310 | sorted_list = sorted(err_list, key=lambda x: x[0], reverse=True) |
| 311 | err_group = [len(sorted_list), suite, test, err_msg, sorted_list] |
| 312 | error_groups.append(err_group) |
| 313 | |
| 314 | # Sort the errors by the number of errors of each type. Then output them in |
| 315 | # order. |
| 316 | sorted_errors = sorted(error_groups, key=lambda x: x[0], reverse=True) |
| 317 | for i in range(0, len(sorted_errors)): |
| 318 | err_group = sorted_errors[i] |
| 319 | suite = err_group[1] |
| 320 | test = err_group[2] |
| 321 | err_msg = err_group[3] |
| 322 | err_list = err_group[4] |
| 323 | out_file.write('Suite: %s\n' % suite) |
| 324 | out_file.write(' %s (%d failures)\n' % (test, len(err_list))) |
| 325 | out_file.write(' (%s)\n' % err_msg) |
| 326 | for i in range(0, len(err_list)): |
| 327 | err = err_list[i] |
| 328 | out_file.write(' %s, %s, %s\n' % (format_date(err[0]), err[1], |
| 329 | err[2])) |
| 330 | out_file.write('\n') |
| 331 | |
| 332 | print('Report generated in %s.' % filename) |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 333 | return filename |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 334 | |
| 335 | |
| 336 | def UpdateReport(report_dict, builder, test, report_date, build_link, |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 337 | test_summary, board, color): |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 338 | """Update the data in our report dictionary with current test's data.""" |
| 339 | |
| 340 | if 'date' not in report_dict: |
| 341 | report_dict['date'] = [report_date] |
| 342 | elif report_date not in report_dict['date']: |
| 343 | # It is possible that some of the builders started/finished on different |
| 344 | # days, so we allow for multiple dates in the reports. |
| 345 | report_dict['date'].append(report_date) |
| 346 | |
| 347 | build_key = '' |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 348 | if builder == GCC_ROTATING_BUILDER: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 349 | build_key = '%s-gcc-toolchain' % board |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 350 | elif builder == LLVM_ROTATING_BUILDER: |
| 351 | build_key = '%s-llvm-next-toolchain' % board |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 352 | else: |
| 353 | build_key = builder |
| 354 | |
| 355 | if build_key not in report_dict.keys(): |
| 356 | build_dict = dict() |
| 357 | else: |
| 358 | build_dict = report_dict[build_key] |
| 359 | |
| 360 | if 'build_link' not in build_dict: |
| 361 | build_dict['build_link'] = build_link |
| 362 | |
| 363 | if 'date' not in build_dict: |
| 364 | build_dict['date'] = report_date |
| 365 | |
| 366 | if 'board' in build_dict and build_dict['board'] != board: |
Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 367 | raise RuntimeError( |
| 368 | 'Error: Two different boards (%s,%s) in one build (%s)!' % |
| 369 | (board, build_dict['board'], build_link)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 370 | build_dict['board'] = board |
| 371 | |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 372 | color_key = '%s-color' % test |
| 373 | build_dict[color_key] = color |
| 374 | |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 375 | # Check to see if we already have a build status for this build_key |
| 376 | status = '' |
| 377 | if 'build_status' in build_dict.keys(): |
| 378 | # Use current build_status, unless current test failed (see below). |
| 379 | status = build_dict['build_status'] |
| 380 | |
| 381 | if not test_summary: |
| 382 | # Current test data was not available, so something was bad with build. |
| 383 | build_dict['build_status'] = 'bad' |
| 384 | build_dict[test] = '[ no data ]' |
| 385 | else: |
| 386 | build_dict[test] = test_summary |
| 387 | if not status: |
| 388 | # Current test ok; no other data, so assume build was ok. |
| 389 | build_dict['build_status'] = 'ok' |
| 390 | |
| 391 | report_dict[build_key] = build_dict |
| 392 | |
| 393 | |
| 394 | def UpdateBuilds(builds): |
| 395 | """Update the data in our build-data.txt file.""" |
| 396 | |
| 397 | # The build data file records the last build number for which we |
| 398 | # generated a report. When we generate the next report, we read |
| 399 | # this data and increment it to get the new data; when we finish |
| 400 | # generating the reports, we write the updated values into this file. |
| 401 | # NOTE: One side effect of doing this at the end: If the script |
| 402 | # fails in the middle of generating a report, this data does not get |
| 403 | # updated. |
| 404 | with open(BUILD_DATA_FILE, 'w') as fp: |
| 405 | gcc_max = 0 |
| 406 | llvm_max = 0 |
| 407 | for b in builds: |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 408 | if b[0] == GCC_ROTATING_BUILDER: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 409 | gcc_max = max(gcc_max, b[1]) |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 410 | elif b[0] == LLVM_ROTATING_BUILDER: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 411 | llvm_max = max(llvm_max, b[1]) |
| 412 | else: |
| 413 | fp.write('%s,%d\n' % (b[0], b[1])) |
| 414 | if gcc_max > 0: |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 415 | fp.write('%s,%d\n' % (GCC_ROTATING_BUILDER, gcc_max)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 416 | if llvm_max > 0: |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 417 | fp.write('%s,%d\n' % (LLVM_ROTATING_BUILDER, llvm_max)) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 418 | |
| 419 | |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 420 | def GetBuilds(date=0): |
| 421 | """Get build id from builds.""" |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 422 | |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 423 | # If date is set, get the build id from waterfall. |
| 424 | builds = [] |
| 425 | |
| 426 | if date: |
| 427 | for builder in WATERFALL_BUILDERS + ROTATING_BUILDERS: |
| 428 | build_ids = GetBuildID(builder, date) |
| 429 | for build_id in build_ids: |
| 430 | builds.append((builder, build_id)) |
| 431 | return builds |
| 432 | |
| 433 | # If date is not set, we try to get the most recent builds. |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 434 | # Read the values of the last builds used to generate a report, and |
| 435 | # increment them appropriately, to get values for generating the |
| 436 | # current report. (See comments in UpdateBuilds). |
| 437 | with open(BUILD_DATA_FILE, 'r') as fp: |
| 438 | lines = fp.readlines() |
| 439 | |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 440 | for l in lines: |
| 441 | l = l.rstrip() |
| 442 | words = l.split(',') |
| 443 | builder = words[0] |
| 444 | build = int(words[1]) |
| 445 | builds.append((builder, build + 1)) |
| 446 | # NOTE: We are assuming here that there are always 2 daily builds in |
| 447 | # each of the rotating builders. I am not convinced this is a valid |
| 448 | # assumption. |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 449 | if builder in ROTATING_BUILDERS: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 450 | builds.append((builder, build + 2)) |
| 451 | |
| 452 | return builds |
| 453 | |
| 454 | |
| 455 | def RecordFailures(failure_dict, platform, suite, builder, int_date, log_file, |
| 456 | build_num, failed): |
| 457 | """Read and update the stored data about test failures.""" |
| 458 | |
| 459 | # Get the dictionary for this particular test suite from the failures |
| 460 | # dictionary. |
| 461 | suite_dict = failure_dict[suite] |
| 462 | |
| 463 | # Read in the entire log file for this test/build. |
| 464 | with open(log_file, 'r') as in_file: |
| 465 | lines = in_file.readlines() |
| 466 | |
| 467 | # Update the entries in the failure dictionary for each test within this suite |
| 468 | # that failed. |
| 469 | for test in failed: |
| 470 | # Check to see if there is already an entry in the suite dictionary for this |
| 471 | # test; if so use that, otherwise create a new entry. |
| 472 | if test in suite_dict: |
| 473 | test_dict = suite_dict[test] |
| 474 | else: |
| 475 | test_dict = dict() |
| 476 | # Parse the lines from the log file, looking for lines that indicate this |
| 477 | # test failed. |
| 478 | msg = '' |
| 479 | for l in lines: |
| 480 | words = l.split() |
| 481 | if len(words) < 3: |
| 482 | continue |
| 483 | if ((words[0] == test and words[1] == 'ERROR:') or |
| 484 | (words[0] == 'provision' and words[1] == 'FAIL:')): |
| 485 | words = words[2:] |
| 486 | # Get the error message for the failure. |
| 487 | msg = ' '.join(words) |
| 488 | if not msg: |
| 489 | msg = 'Unknown_Error' |
| 490 | |
| 491 | # Look for an existing entry for this error message in the test dictionary. |
| 492 | # If found use that, otherwise create a new entry for this error message. |
| 493 | if msg in test_dict: |
| 494 | error_list = test_dict[msg] |
| 495 | else: |
| 496 | error_list = list() |
| 497 | # Create an entry for this new failure |
| 498 | new_item = [int_date, platform, builder, build_num] |
| 499 | # Add this failure to the error list if it's not already there. |
| 500 | if new_item not in error_list: |
| 501 | error_list.append([int_date, platform, builder, build_num]) |
| 502 | # Sort the error list by date. |
| 503 | error_list.sort(key=lambda x: x[0]) |
| 504 | # Calculate the earliest date to save; delete records for older failures. |
| 505 | earliest_date = int_date - MAX_SAVE_RECORDS |
| 506 | i = 0 |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 507 | while i < len(error_list) and error_list[i][0] <= earliest_date: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 508 | i += 1 |
| 509 | if i > 0: |
| 510 | error_list = error_list[i:] |
| 511 | # Save the error list in the test's dictionary, keyed on error_msg. |
| 512 | test_dict[msg] = error_list |
| 513 | |
| 514 | # Save the updated test dictionary in the test_suite dictionary. |
| 515 | suite_dict[test] = test_dict |
| 516 | |
| 517 | # Save the updated test_suite dictionary in the failure dictionary. |
| 518 | failure_dict[suite] = suite_dict |
| 519 | |
| 520 | |
| 521 | def ParseLogFile(log_file, test_data_dict, failure_dict, test, builder, |
| 522 | build_num, build_link): |
| 523 | """Parse the log file from the given builder, build_num and test. |
| 524 | |
| 525 | Also adds the results for this test to our test results dictionary, |
| 526 | and calls RecordFailures, to update our test failure data. |
| 527 | """ |
| 528 | |
| 529 | lines = [] |
| 530 | with open(log_file, 'r') as infile: |
| 531 | lines = infile.readlines() |
| 532 | |
| 533 | passed = {} |
| 534 | failed = {} |
| 535 | not_run = {} |
| 536 | date = '' |
| 537 | status = '' |
| 538 | board = '' |
| 539 | num_provision_errors = 0 |
| 540 | build_ok = True |
| 541 | afe_line = '' |
| 542 | |
| 543 | for line in lines: |
| 544 | if line.rstrip() == '<title>404 Not Found</title>': |
| 545 | print('Warning: File for %s (build number %d), %s was not found.' % |
| 546 | (builder, build_num, test)) |
| 547 | build_ok = False |
| 548 | break |
| 549 | if '[ PASSED ]' in line: |
| 550 | test_name = line.split()[0] |
| 551 | if test_name != 'Suite': |
| 552 | passed[test_name] = True |
| 553 | elif '[ FAILED ]' in line: |
| 554 | test_name = line.split()[0] |
| 555 | if test_name == 'provision': |
| 556 | num_provision_errors += 1 |
| 557 | not_run[test_name] = True |
| 558 | elif test_name != 'Suite': |
| 559 | failed[test_name] = True |
| 560 | elif line.startswith('started: '): |
| 561 | date = line.rstrip() |
| 562 | date = date[9:] |
| 563 | date_obj = time.strptime(date, '%a %b %d %H:%M:%S %Y') |
| 564 | int_date = ( |
| 565 | date_obj.tm_year * 10000 + date_obj.tm_mon * 100 + date_obj.tm_mday) |
| 566 | date = time.strftime('%a %b %d %Y', date_obj) |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 567 | elif not status and line.startswith('status: '): |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 568 | status = line.rstrip() |
| 569 | words = status.split(':') |
| 570 | status = words[-1] |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 571 | elif line.find('Suite passed with a warning') != -1: |
| 572 | status = 'WARNING' |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 573 | elif line.startswith('@@@STEP_LINK@Link to suite@'): |
| 574 | afe_line = line.rstrip() |
| 575 | words = afe_line.split('@') |
| 576 | for w in words: |
| 577 | if w.startswith('http'): |
| 578 | afe_line = w |
| 579 | afe_line = afe_line.replace('&', '&') |
| 580 | elif 'INFO: RunCommand:' in line: |
| 581 | words = line.split() |
| 582 | for i in range(0, len(words) - 1): |
| 583 | if words[i] == '--board': |
| 584 | board = words[i + 1] |
| 585 | |
| 586 | test_dict = test_data_dict[test] |
| 587 | test_list = test_dict['tests'] |
| 588 | |
| 589 | if build_ok: |
| 590 | for t in test_list: |
| 591 | if not t in passed and not t in failed: |
| 592 | not_run[t] = True |
| 593 | |
| 594 | total_pass = len(passed) |
| 595 | total_fail = len(failed) |
| 596 | total_notrun = len(not_run) |
| 597 | |
| 598 | else: |
| 599 | total_pass = 0 |
| 600 | total_fail = 0 |
| 601 | total_notrun = 0 |
| 602 | status = 'Not found.' |
| 603 | if not build_ok: |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 604 | return [], date, board, 0, ' ' |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 605 | |
| 606 | build_dict = dict() |
| 607 | build_dict['id'] = build_num |
| 608 | build_dict['builder'] = builder |
| 609 | build_dict['date'] = date |
| 610 | build_dict['build_link'] = build_link |
| 611 | build_dict['total_pass'] = total_pass |
| 612 | build_dict['total_fail'] = total_fail |
| 613 | build_dict['total_not_run'] = total_notrun |
| 614 | build_dict['afe_job_link'] = afe_line |
| 615 | build_dict['provision_errors'] = num_provision_errors |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 616 | if status.strip() == 'SUCCESS': |
| 617 | build_dict['color'] = 'green ' |
| 618 | elif status.strip() == 'FAILURE': |
| 619 | build_dict['color'] = ' red ' |
| 620 | elif status.strip() == 'WARNING': |
| 621 | build_dict['color'] = 'orange' |
| 622 | else: |
| 623 | build_dict['color'] = ' ' |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 624 | |
| 625 | # Use YYYYMMDD (integer) as the build record key |
| 626 | if build_ok: |
| 627 | if board in test_dict: |
| 628 | board_dict = test_dict[board] |
| 629 | else: |
| 630 | board_dict = dict() |
| 631 | board_dict[int_date] = build_dict |
| 632 | |
| 633 | # Only keep the last 5 records (based on date) |
| 634 | keys_list = board_dict.keys() |
| 635 | if len(keys_list) > MAX_SAVE_RECORDS: |
| 636 | min_key = min(keys_list) |
| 637 | del board_dict[min_key] |
| 638 | |
| 639 | # Make sure changes get back into the main dictionary |
| 640 | test_dict[board] = board_dict |
| 641 | test_data_dict[test] = test_dict |
| 642 | |
| 643 | if len(failed) > 0: |
| 644 | RecordFailures(failure_dict, board, test, builder, int_date, log_file, |
| 645 | build_num, failed) |
| 646 | |
| 647 | summary_result = '[%2d/ %2d/ %2d]' % (total_pass, total_fail, total_notrun) |
| 648 | |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 649 | return summary_result, date, board, int_date, build_dict['color'] |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 650 | |
| 651 | |
| 652 | def DownloadLogFile(builder, buildnum, test, test_family): |
| 653 | |
| 654 | ce = command_executer.GetCommandExecuter() |
| 655 | os.system('mkdir -p %s/%s/%s' % (DOWNLOAD_DIR, builder, test)) |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 656 | if builder in ROTATING_BUILDERS: |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 657 | source = ('https://uberchromegw.corp.google.com/i/chromiumos.tryserver' |
| 658 | '/builders/%s/builds/%d/steps/%s%%20%%5B%s%%5D/logs/stdio' % |
| 659 | (builder, buildnum, test_family, test)) |
| 660 | build_link = ('https://uberchromegw.corp.google.com/i/chromiumos.tryserver' |
| 661 | '/builders/%s/builds/%d' % (builder, buildnum)) |
| 662 | else: |
| 663 | source = ('https://uberchromegw.corp.google.com/i/chromeos/builders/%s/' |
| 664 | 'builds/%d/steps/%s%%20%%5B%s%%5D/logs/stdio' % |
| 665 | (builder, buildnum, test_family, test)) |
| 666 | build_link = ('https://uberchromegw.corp.google.com/i/chromeos/builders/%s' |
| 667 | '/builds/%d' % (builder, buildnum)) |
| 668 | |
| 669 | target = '%s/%s/%s/%d' % (DOWNLOAD_DIR, builder, test, buildnum) |
| 670 | if not os.path.isfile(target) or os.path.getsize(target) == 0: |
| 671 | cmd = 'sso_client %s > %s' % (source, target) |
| 672 | status = ce.RunCommand(cmd) |
| 673 | if status != 0: |
| 674 | return '', '' |
| 675 | |
| 676 | return target, build_link |
| 677 | |
| 678 | |
Manoj Gupta | 6382452 | 2016-12-14 11:05:18 -0800 | [diff] [blame] | 679 | # Check for prodaccess. |
| 680 | def CheckProdAccess(): |
| 681 | status, output, _ = command_executer.GetCommandExecuter().RunCommandWOutput( |
| 682 | 'prodcertstatus') |
| 683 | if status != 0: |
| 684 | return False |
| 685 | # Verify that status is not expired |
| 686 | if 'expires' in output: |
| 687 | return True |
| 688 | return False |
| 689 | |
| 690 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 691 | def ValidOptions(parser, options): |
| 692 | too_many_options = False |
| 693 | if options.main: |
| 694 | if options.rotating or options.failures_report: |
| 695 | too_many_options = True |
| 696 | elif options.rotating and options.failures_report: |
| 697 | too_many_options = True |
| 698 | |
| 699 | if too_many_options: |
| 700 | parser.error('Can only specify one of --main, --rotating or' |
| 701 | ' --failures_report.') |
| 702 | |
| 703 | conflicting_failure_options = False |
| 704 | if options.failures_report and options.omit_failures: |
| 705 | conflicting_failure_options = True |
| 706 | parser.error('Cannot specify both --failures_report and --omit_failures.') |
| 707 | |
| 708 | return not too_many_options and not conflicting_failure_options |
| 709 | |
| 710 | |
| 711 | def Main(argv): |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 712 | """Main function for this script.""" |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 713 | parser = argparse.ArgumentParser() |
| 714 | parser.add_argument( |
| 715 | '--main', |
| 716 | dest='main', |
| 717 | default=False, |
| 718 | action='store_true', |
| 719 | help='Generate report only for main waterfall ' |
| 720 | 'builders.') |
| 721 | parser.add_argument( |
| 722 | '--rotating', |
| 723 | dest='rotating', |
| 724 | default=False, |
| 725 | action='store_true', |
| 726 | help='Generate report only for rotating builders.') |
| 727 | parser.add_argument( |
| 728 | '--failures_report', |
| 729 | dest='failures_report', |
| 730 | default=False, |
| 731 | action='store_true', |
| 732 | help='Only generate the failures section of the report.') |
| 733 | parser.add_argument( |
| 734 | '--omit_failures', |
| 735 | dest='omit_failures', |
| 736 | default=False, |
| 737 | action='store_true', |
| 738 | help='Do not generate the failures section of the report.') |
| 739 | parser.add_argument( |
| 740 | '--no_update', |
| 741 | dest='no_update', |
| 742 | default=False, |
| 743 | action='store_true', |
| 744 | help='Run reports, but do not update the data files.') |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 745 | parser.add_argument( |
| 746 | '--date', |
| 747 | dest='date', |
| 748 | default=0, |
| 749 | type=int, |
| 750 | help='The date YYYYMMDD of waterfall report.') |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 751 | |
| 752 | options = parser.parse_args(argv) |
| 753 | |
| 754 | if not ValidOptions(parser, options): |
| 755 | return 1 |
| 756 | |
| 757 | main_only = options.main |
| 758 | rotating_only = options.rotating |
| 759 | failures_report = options.failures_report |
| 760 | omit_failures = options.omit_failures |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 761 | date = options.date |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 762 | |
| 763 | test_data_dict = dict() |
| 764 | failure_dict = dict() |
Manoj Gupta | 6382452 | 2016-12-14 11:05:18 -0800 | [diff] [blame] | 765 | |
| 766 | prod_access = CheckProdAccess() |
| 767 | if not prod_access: |
| 768 | print('ERROR: Please run prodaccess first.') |
| 769 | return |
| 770 | |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 771 | with open('%s/waterfall-test-data.json' % DATA_DIR, 'r') as input_file: |
| 772 | test_data_dict = json.load(input_file) |
| 773 | |
| 774 | with open('%s/test-failure-data.json' % DATA_DIR, 'r') as fp: |
| 775 | failure_dict = json.load(fp) |
| 776 | |
Yunlian Jiang | 6b8bacb | 2016-12-15 14:53:39 -0800 | [diff] [blame] | 777 | builds = GetBuilds(date) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 778 | |
| 779 | waterfall_report_dict = dict() |
| 780 | rotating_report_dict = dict() |
| 781 | int_date = 0 |
| 782 | for test_desc in TESTS: |
| 783 | test, test_family = test_desc |
| 784 | for build in builds: |
| 785 | (builder, buildnum) = build |
| 786 | if test.startswith('kernel') and 'llvm' in builder: |
| 787 | continue |
| 788 | if 'x86' in builder and not test.startswith('bvt'): |
| 789 | continue |
| 790 | target, build_link = DownloadLogFile(builder, buildnum, test, test_family) |
| 791 | |
| 792 | if os.path.exists(target): |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 793 | test_summary, report_date, board, tmp_date, color = ParseLogFile( |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 794 | target, test_data_dict, failure_dict, test, builder, buildnum, |
| 795 | build_link) |
| 796 | |
| 797 | if tmp_date != 0: |
| 798 | int_date = tmp_date |
| 799 | |
| 800 | if builder in ROTATING_BUILDERS: |
| 801 | UpdateReport(rotating_report_dict, builder, test, report_date, |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 802 | build_link, test_summary, board, color) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 803 | else: |
| 804 | UpdateReport(waterfall_report_dict, builder, test, report_date, |
Caroline Tice | f0ad65c | 2016-11-29 10:40:23 -0800 | [diff] [blame] | 805 | build_link, test_summary, board, color) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 806 | |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 807 | PruneOldFailures(failure_dict, int_date) |
| 808 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 809 | if waterfall_report_dict and not rotating_only and not failures_report: |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 810 | main_report = GenerateWaterfallReport(waterfall_report_dict, failure_dict, |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 811 | 'main', int_date, omit_failures) |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 812 | EmailReport(main_report, 'Main', format_date(int_date)) |
| 813 | shutil.copy(main_report, ARCHIVE_DIR) |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 814 | if rotating_report_dict and not main_only and not failures_report: |
Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 815 | rotating_report = GenerateWaterfallReport( |
| 816 | rotating_report_dict, failure_dict, 'rotating', int_date, omit_failures) |
Caroline Tice | e02e9f8 | 2016-12-01 13:14:41 -0800 | [diff] [blame] | 817 | EmailReport(rotating_report, 'Rotating', format_date(int_date)) |
| 818 | shutil.copy(rotating_report, ARCHIVE_DIR) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 819 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 820 | if failures_report: |
| 821 | failures_report = GenerateFailuresReport(failure_dict, int_date) |
| 822 | EmailReport(failures_report, 'Failures', format_date(int_date)) |
| 823 | shutil.copy(failures_report, ARCHIVE_DIR) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 824 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 825 | if not options.no_update: |
| 826 | with open('%s/waterfall-test-data.json' % DATA_DIR, 'w') as out_file: |
| 827 | json.dump(test_data_dict, out_file, indent=2) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 828 | |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 829 | with open('%s/test-failure-data.json' % DATA_DIR, 'w') as out_file: |
| 830 | json.dump(failure_dict, out_file, indent=2) |
| 831 | |
| 832 | UpdateBuilds(builds) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 833 | |
| 834 | |
| 835 | if __name__ == '__main__': |
Caroline Tice | faa3c55 | 2016-12-13 11:29:59 -0800 | [diff] [blame] | 836 | Main(sys.argv[1:]) |
Caroline Tice | 4846206 | 2016-11-18 16:49:00 -0800 | [diff] [blame] | 837 | sys.exit(0) |