Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 2 | # Copyright 2016 gRPC authors. |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 3 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 7 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 9 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 15 | |
| 16 | """Tool to get build statistics from Jenkins and upload to BigQuery.""" |
| 17 | |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 18 | from __future__ import print_function |
| 19 | |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 20 | import argparse |
| 21 | import jenkinsapi |
| 22 | from jenkinsapi.custom_exceptions import JenkinsAPIException |
| 23 | from jenkinsapi.jenkins import Jenkins |
| 24 | import json |
| 25 | import os |
| 26 | import re |
| 27 | import sys |
| 28 | import urllib |
| 29 | |
| 30 | |
| 31 | gcp_utils_dir = os.path.abspath(os.path.join( |
| 32 | os.path.dirname(__file__), '../gcp/utils')) |
| 33 | sys.path.append(gcp_utils_dir) |
| 34 | import big_query_utils |
| 35 | |
| 36 | |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 37 | _PROJECT_ID = 'grpc-testing' |
| 38 | _HAS_MATRIX = True |
Matt Kwong | 983c2da | 2016-11-10 15:07:59 -0800 | [diff] [blame] | 39 | _BUILDS = {'gRPC_interop_master': not _HAS_MATRIX, |
| 40 | 'gRPC_master_linux': not _HAS_MATRIX, |
| 41 | 'gRPC_master_macos': not _HAS_MATRIX, |
| 42 | 'gRPC_master_windows': not _HAS_MATRIX, |
| 43 | 'gRPC_performance_master': not _HAS_MATRIX, |
| 44 | 'gRPC_portability_master_linux': not _HAS_MATRIX, |
| 45 | 'gRPC_portability_master_windows': not _HAS_MATRIX, |
| 46 | 'gRPC_master_asanitizer_c': not _HAS_MATRIX, |
| 47 | 'gRPC_master_asanitizer_cpp': not _HAS_MATRIX, |
| 48 | 'gRPC_master_msan_c': not _HAS_MATRIX, |
| 49 | 'gRPC_master_tsanitizer_c': not _HAS_MATRIX, |
| 50 | 'gRPC_master_tsan_cpp': not _HAS_MATRIX, |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 51 | 'gRPC_interop_pull_requests': not _HAS_MATRIX, |
Matt Kwong | 983c2da | 2016-11-10 15:07:59 -0800 | [diff] [blame] | 52 | 'gRPC_performance_pull_requests': not _HAS_MATRIX, |
| 53 | 'gRPC_portability_pull_requests_linux': not _HAS_MATRIX, |
| 54 | 'gRPC_portability_pr_win': not _HAS_MATRIX, |
| 55 | 'gRPC_pull_requests_linux': not _HAS_MATRIX, |
| 56 | 'gRPC_pull_requests_macos': not _HAS_MATRIX, |
| 57 | 'gRPC_pr_win': not _HAS_MATRIX, |
| 58 | 'gRPC_pull_requests_asan_c': not _HAS_MATRIX, |
| 59 | 'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX, |
| 60 | 'gRPC_pull_requests_msan_c': not _HAS_MATRIX, |
| 61 | 'gRPC_pull_requests_tsan_c': not _HAS_MATRIX, |
| 62 | 'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX, |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 63 | } |
| 64 | _URL_BASE = 'https://grpc-testing.appspot.com/job' |
Adele Zhou | 445534e | 2016-09-06 17:57:11 -0700 | [diff] [blame] | 65 | |
| 66 | # This is a dynamic list where known and active issues should be added. |
| 67 | # Fixed ones should be removed. |
| 68 | # Also try not to add multiple messages from the same failure. |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 69 | _KNOWN_ERRORS = [ |
| 70 | 'Failed to build workspace Tests with scheme AllTests', |
| 71 | 'Build timed out', |
Adele Zhou | 859d2a4 | 2016-09-16 15:15:44 -0700 | [diff] [blame] | 72 | 'TIMEOUT: tools/run_tests/pre_build_node.sh', |
| 73 | 'TIMEOUT: tools/run_tests/pre_build_ruby.sh', |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 74 | 'FATAL: Unable to produce a script file', |
Adele Zhou | 107e51e | 2016-09-08 11:37:30 -0700 | [diff] [blame] | 75 | 'FAILED: build_docker_c\+\+', |
Adele Zhou | 445534e | 2016-09-06 17:57:11 -0700 | [diff] [blame] | 76 | 'cannot find package \"cloud.google.com/go/compute/metadata\"', |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 77 | 'LLVM ERROR: IO failure on output stream.', |
| 78 | 'MSBUILD : error MSB1009: Project file does not exist.', |
Adele Zhou | 445534e | 2016-09-06 17:57:11 -0700 | [diff] [blame] | 79 | 'fatal: git fetch_pack: expected ACK/NAK', |
| 80 | 'Failed to fetch from http://github.com/grpc/grpc.git', |
| 81 | ('hudson.remoting.RemotingSystemException: java.io.IOException: ' |
| 82 | 'Backing channel is disconnected.'), |
Adele Zhou | 107e51e | 2016-09-08 11:37:30 -0700 | [diff] [blame] | 83 | 'hudson.remoting.ChannelClosedException', |
Adele Zhou | 59af64b | 2016-09-22 15:23:54 -0700 | [diff] [blame] | 84 | 'Could not initialize class hudson.Util', |
| 85 | 'Too many open files in system', |
Adele Zhou | 445534e | 2016-09-06 17:57:11 -0700 | [diff] [blame] | 86 | 'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll', |
| 87 | 'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy', |
| 88 | 'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll', |
| 89 | ('tests.bins/asan/h2_proxy_test streaming_error_response ' |
| 90 | 'GRPC_POLL_STRATEGY=legacy'), |
Adele Zhou | 566c4c0 | 2017-08-10 15:51:47 -0700 | [diff] [blame] | 91 | 'hudson.plugins.git.GitException', |
| 92 | 'Couldn\'t find any revision to build', |
| 93 | 'org.jenkinsci.plugin.Diskcheck.preCheckout', |
| 94 | 'Something went wrong while deleting Files', |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 95 | ] |
Adele Zhou | c210147 | 2017-08-09 17:04:27 -0700 | [diff] [blame] | 96 | _NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.' |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 97 | _UNKNOWN_ERROR = 'Unknown error' |
| 98 | _DATASET_ID = 'build_statistics' |
| 99 | |
| 100 | |
| 101 | def _scrape_for_known_errors(html): |
| 102 | error_list = [] |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 103 | for known_error in _KNOWN_ERRORS: |
| 104 | errors = re.findall(known_error, html) |
| 105 | this_error_count = len(errors) |
| 106 | if this_error_count > 0: |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 107 | error_list.append({'description': known_error, |
| 108 | 'count': this_error_count}) |
| 109 | print('====> %d failures due to %s' % (this_error_count, known_error)) |
Adele Zhou | c210147 | 2017-08-09 17:04:27 -0700 | [diff] [blame] | 110 | return error_list |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 111 | |
| 112 | |
Matt Kwong | 983c2da | 2016-11-10 15:07:59 -0800 | [diff] [blame] | 113 | def _no_report_files_found(html): |
| 114 | return _NO_REPORT_FILES_FOUND_ERROR in html |
| 115 | |
| 116 | |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 117 | def _get_last_processed_buildnumber(build_name): |
| 118 | query = 'SELECT max(build_number) FROM [%s:%s.%s];' % ( |
| 119 | _PROJECT_ID, _DATASET_ID, build_name) |
| 120 | query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query) |
| 121 | page = bq.jobs().getQueryResults( |
| 122 | pageToken=None, |
| 123 | **query_job['jobReference']).execute(num_retries=3) |
| 124 | if page['rows'][0]['f'][0]['v']: |
| 125 | return int(page['rows'][0]['f'][0]['v']) |
| 126 | return 0 |
| 127 | |
| 128 | |
| 129 | def _process_matrix(build, url_base): |
| 130 | matrix_list = [] |
| 131 | for matrix in build.get_matrix_runs(): |
| 132 | matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*', |
| 133 | matrix.name).groups()[0] |
| 134 | matrix_tuple = matrix_str.split(',') |
| 135 | json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % ( |
| 136 | url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) |
| 137 | console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % ( |
| 138 | url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) |
| 139 | matrix_dict = {'name': matrix_str, |
| 140 | 'duration': matrix.get_duration().total_seconds()} |
| 141 | matrix_dict.update(_process_build(json_url, console_url)) |
| 142 | matrix_list.append(matrix_dict) |
| 143 | |
| 144 | return matrix_list |
| 145 | |
| 146 | |
| 147 | def _process_build(json_url, console_url): |
| 148 | build_result = {} |
| 149 | error_list = [] |
| 150 | try: |
| 151 | html = urllib.urlopen(json_url).read() |
| 152 | test_result = json.loads(html) |
| 153 | print('====> Parsing result from %s' % json_url) |
| 154 | failure_count = test_result['failCount'] |
| 155 | build_result['pass_count'] = test_result['passCount'] |
| 156 | build_result['failure_count'] = failure_count |
Adele Zhou | 87c38e9 | 2017-08-03 11:39:51 -0700 | [diff] [blame] | 157 | # This means Jenkins failure occurred. |
Matt Kwong | 983c2da | 2016-11-10 15:07:59 -0800 | [diff] [blame] | 158 | build_result['no_report_files_found'] = _no_report_files_found(html) |
Adele Zhou | 87c38e9 | 2017-08-03 11:39:51 -0700 | [diff] [blame] | 159 | # Only check errors if Jenkins failure occurred. |
| 160 | if build_result['no_report_files_found']: |
Adele Zhou | c210147 | 2017-08-09 17:04:27 -0700 | [diff] [blame] | 161 | error_list = _scrape_for_known_errors(html) |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 162 | except Exception as e: |
| 163 | print('====> Got exception for %s: %s.' % (json_url, str(e))) |
| 164 | print('====> Parsing errors from %s.' % console_url) |
| 165 | html = urllib.urlopen(console_url).read() |
| 166 | build_result['pass_count'] = 0 |
| 167 | build_result['failure_count'] = 1 |
Adele Zhou | 566c4c0 | 2017-08-10 15:51:47 -0700 | [diff] [blame] | 168 | # In this case, the string doesn't exist in the result html but the fact |
| 169 | # that we fail to parse the result html indicates Jenkins failure and hence |
| 170 | # no report files were generated. |
Adele Zhou | c210147 | 2017-08-09 17:04:27 -0700 | [diff] [blame] | 171 | build_result['no_report_files_found'] = True |
| 172 | error_list = _scrape_for_known_errors(html) |
| 173 | |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 174 | if error_list: |
| 175 | build_result['error'] = error_list |
Adele Zhou | c210147 | 2017-08-09 17:04:27 -0700 | [diff] [blame] | 176 | elif build_result['no_report_files_found']: |
| 177 | build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}] |
Adele Zhou | 87c38e9 | 2017-08-03 11:39:51 -0700 | [diff] [blame] | 178 | else: |
| 179 | build_result['error'] = [{'description': '', 'count': 0}] |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 180 | |
| 181 | return build_result |
| 182 | |
| 183 | |
| 184 | # parse command line |
| 185 | argp = argparse.ArgumentParser(description='Get build statistics.') |
| 186 | argp.add_argument('-u', '--username', default='jenkins') |
| 187 | argp.add_argument('-b', '--builds', |
| 188 | choices=['all'] + sorted(_BUILDS.keys()), |
| 189 | nargs='+', |
| 190 | default=['all']) |
| 191 | args = argp.parse_args() |
| 192 | |
| 193 | J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken') |
| 194 | bq = big_query_utils.create_big_query() |
| 195 | |
| 196 | for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: |
| 197 | print('====> Build: %s' % build_name) |
| 198 | # Since get_last_completed_build() always fails due to malformatted string |
| 199 | # error, we use get_build_metadata() instead. |
| 200 | job = None |
| 201 | try: |
| 202 | job = J[build_name] |
| 203 | except Exception as e: |
| 204 | print('====> Failed to get build %s: %s.' % (build_name, str(e))) |
| 205 | continue |
| 206 | last_processed_build_number = _get_last_processed_buildnumber(build_name) |
| 207 | last_complete_build_number = job.get_last_completed_buildnumber() |
| 208 | # To avoid processing all builds for a project never looked at. In this case, |
| 209 | # only examine 10 latest builds. |
| 210 | starting_build_number = max(last_processed_build_number+1, |
| 211 | last_complete_build_number-9) |
| 212 | for build_number in xrange(starting_build_number, |
| 213 | last_complete_build_number+1): |
| 214 | print('====> Processing %s build %d.' % (build_name, build_number)) |
| 215 | build = None |
| 216 | try: |
| 217 | build = job.get_build_metadata(build_number) |
Adele Zhou | 566c4c0 | 2017-08-10 15:51:47 -0700 | [diff] [blame] | 218 | print('====> Build status: %s.' % build.get_status()) |
| 219 | if build.get_status() == 'ABORTED': |
| 220 | continue |
| 221 | # If any build is still running, stop processing this job. Next time, we |
| 222 | # start from where it was left so that all builds are processed |
| 223 | # sequentially. |
| 224 | if build.is_running(): |
| 225 | print('====> Build %d is still running.' % build_number) |
| 226 | break |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 227 | except KeyError: |
| 228 | print('====> Build %s is missing. Skip.' % build_number) |
| 229 | continue |
| 230 | build_result = {'build_number': build_number, |
| 231 | 'timestamp': str(build.get_timestamp())} |
| 232 | url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number) |
| 233 | if _BUILDS[build_name]: # The build has matrix, such as gRPC_master. |
| 234 | build_result['matrix'] = _process_matrix(build, url_base) |
| 235 | else: |
| 236 | json_url = '%s/testReport/api/json' % url_base |
| 237 | console_url = '%s/consoleFull' % url_base |
| 238 | build_result['duration'] = build.get_duration().total_seconds() |
Adele Zhou | 566c4c0 | 2017-08-10 15:51:47 -0700 | [diff] [blame] | 239 | build_stat = _process_build(json_url, console_url) |
| 240 | build_result.update(build_stat) |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 241 | rows = [big_query_utils.make_row(build_number, build_result)] |
| 242 | if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, |
| 243 | rows): |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 244 | print('====> Error uploading result to bigquery.') |
Adele Zhou | bf3b769 | 2016-08-11 18:45:18 -0700 | [diff] [blame] | 245 | sys.exit(1) |