Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 1 | #!/usr/bin/python2 |
| 2 | """Script for running nightly compiler tests on ChromeOS. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 3 | |
| 4 | This script launches a buildbot to build ChromeOS with the latest compiler on |
| 5 | a particular board; then it finds and downloads the trybot image and the |
| 6 | corresponding official image, and runs crosperf performance tests comparing |
| 7 | the two. It then generates a report, emails it to the c-compiler-chrome, as |
| 8 | well as copying the images into the seven-day reports directory. |
| 9 | """ |
| 10 | |
| 11 | # Script to test different toolchains against ChromeOS benchmarks. |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 12 | |
| 13 | from __future__ import print_function |
| 14 | |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 15 | import datetime |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 16 | import optparse |
| 17 | import os |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 18 | import re |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 19 | import sys |
| 20 | import time |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 21 | |
| 22 | from utils import command_executer |
| 23 | from utils import logger |
| 24 | |
| 25 | from utils import buildbot_utils |
| 26 | |
| 27 | # CL that updated GCC ebuilds to use 'next_gcc'. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 28 | USE_NEXT_GCC_PATCH = '230260' |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 29 | |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 30 | # CL that uses LLVM to build the peppy image. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 31 | USE_LLVM_PATCH = '295217' |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 32 | |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 33 | # The boards on which we run weekly reports |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 34 | WEEKLY_REPORT_BOARDS = ['lumpy'] |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 35 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 36 | CROSTC_ROOT = '/usr/local/google/crostc' |
| 37 | ROLE_ACCOUNT = 'mobiletc-prebuild' |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 38 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 39 | MAIL_PROGRAM = '~/var/bin/mail-sheriff' |
| 40 | WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data') |
| 41 | PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') |
| 42 | NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') |
| 43 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 44 | IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' + |
| 45 | r'{branch}\.{branch_branch}') |
| 46 | TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}' |
| 47 | PFQ_IMAGE_FS = IMAGE_FS + '-rc1' |
| 48 | IMAGE_RE_GROUPS = {'board': r'(?P<board>\S+)', |
| 49 | 'image_type': r'(?P<image_type>\S+)', |
| 50 | 'chrome_version': r'(?P<chrome_version>R\d+)', |
| 51 | 'tip': r'(?P<tip>\d+)', |
| 52 | 'branch': r'(?P<branch>\d+)', |
| 53 | 'branch_branch': r'(?P<branch_branch>\d+)', |
| 54 | 'build_id': r'(?P<build_id>b\d+)'} |
| 55 | TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) |
| 56 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 57 | |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 58 | class ToolchainComparator(object): |
| 59 | """Class for doing the nightly tests work.""" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 60 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 61 | def __init__(self, |
| 62 | board, |
| 63 | remotes, |
| 64 | chromeos_root, |
| 65 | weekday, |
| 66 | patches, |
| 67 | noschedv2=False): |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 68 | self._board = board |
| 69 | self._remotes = remotes |
| 70 | self._chromeos_root = chromeos_root |
| 71 | self._base_dir = os.getcwd() |
| 72 | self._ce = command_executer.GetCommandExecuter() |
| 73 | self._l = logger.GetLogger() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 74 | self._build = '%s-release' % board |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 75 | self._patches = patches.split(',') |
| 76 | self._patches_string = '_'.join(str(p) for p in self._patches) |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 77 | self._noschedv2 = noschedv2 |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 78 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 79 | if not weekday: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 80 | self._weekday = time.strftime('%a') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 81 | else: |
| 82 | self._weekday = weekday |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 83 | timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 84 | '%Y-%m-%d_%H:%M:%S') |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 85 | self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 86 | '%s.%s' % (timestamp, board),) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 87 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 88 | def _GetVanillaImageName(self, trybot_image): |
| 89 | """Given a trybot artifact name, get corresponding vanilla image name. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 90 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 91 | Args: |
| 92 | trybot_image: artifact name such as |
| 93 | 'trybot-daisy-release/R40-6394.0.0-b1389' |
| 94 | |
| 95 | Returns: |
| 96 | Corresponding official image name, e.g. 'daisy-release/R40-6394.0.0'. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 97 | """ |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 98 | mo = re.search(TRYBOT_IMAGE_RE, trybot_image) |
| 99 | assert mo |
| 100 | return IMAGE_FS.replace('\\', '').format(**mo.groupdict()) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 101 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 102 | def _GetNonAFDOImageName(self, trybot_image): |
| 103 | """Given a trybot artifact name, get corresponding non-AFDO image name. |
| 104 | |
| 105 | We get the non-AFDO image from the PFQ builders. This image |
| 106 | is not generated for all the boards and, the closest PFQ image |
| 107 | was the one build for the previous ChromeOS version (the chrome |
| 108 | used in the current version is the one validated in the previous |
| 109 | version). |
| 110 | The previous ChromeOS does not always exist either. So, we try |
| 111 | a couple of versions before. |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 112 | |
| 113 | Args: |
| 114 | trybot_image: artifact name such as |
| 115 | 'trybot-daisy-release/R40-6394.0.0-b1389' |
| 116 | |
| 117 | Returns: |
| 118 | Corresponding chrome PFQ image name, e.g. |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 119 | 'daisy-chrome-pfq/R40-6393.0.0-rc1'. |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 120 | """ |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 121 | mo = re.search(TRYBOT_IMAGE_RE, trybot_image) |
| 122 | assert mo |
| 123 | image_dict = mo.groupdict() |
| 124 | image_dict['image_type'] = 'chrome-pfq' |
| 125 | for _ in xrange(2): |
| 126 | image_dict['tip'] = str(int(image_dict['tip']) - 1) |
| 127 | nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict) |
| 128 | if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image): |
| 129 | return nonafdo_image |
| 130 | return '' |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 131 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 132 | def _FinishSetup(self): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 133 | """Make sure testing_rsa file is properly set up.""" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 134 | # Fix protections on ssh key |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 135 | command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' |
| 136 | '/chrome-src-internal/src/third_party/chromite/ssh_keys' |
| 137 | '/testing_rsa') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 138 | ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 139 | if ret_val != 0: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 140 | raise RuntimeError('chmod for testing_rsa failed') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 141 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 142 | def _TestImages(self, trybot_image, vanilla_image, nonafdo_image): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 143 | """Create crosperf experiment file. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 144 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 145 | Given the names of the trybot, vanilla and non-AFDO images, create the |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 146 | appropriate crosperf experiment file and launch crosperf on it. |
| 147 | """ |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 148 | experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday) |
| 149 | experiment_file_name = '%s_toolchain_experiment.txt' % self._board |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 150 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 151 | compiler_string = 'gcc' |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 152 | if USE_LLVM_PATCH in self._patches_string: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 153 | experiment_file_name = '%s_llvm_experiment.txt' % self._board |
| 154 | compiler_string = 'llvm' |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 155 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 156 | experiment_file = os.path.join(experiment_file_dir, experiment_file_name) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 157 | experiment_header = """ |
| 158 | board: %s |
| 159 | remote: %s |
Luis Lozano | e1efeb8 | 2015-06-16 16:35:44 -0700 | [diff] [blame] | 160 | retries: 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 161 | """ % (self._board, self._remotes) |
| 162 | experiment_tests = """ |
Luis Lozano | 1489d64 | 2015-12-08 10:08:19 -0800 | [diff] [blame] | 163 | benchmark: all_toolchain_perf { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 164 | suite: telemetry_Crosperf |
| 165 | iterations: 3 |
| 166 | } |
| 167 | """ |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 168 | |
| 169 | with open(experiment_file, 'w') as f: |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 170 | f.write(experiment_header) |
| 171 | f.write(experiment_tests) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 172 | |
| 173 | # Now add vanilla to test file. |
| 174 | official_image = """ |
| 175 | vanilla_image { |
| 176 | chromeos_root: %s |
| 177 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 178 | compiler: gcc |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 179 | } |
| 180 | """ % (self._chromeos_root, vanilla_image) |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 181 | f.write(official_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 182 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 183 | # Now add non-AFDO image to test file. |
Luis Lozano | 439f2b7 | 2016-01-08 11:56:02 -0800 | [diff] [blame] | 184 | if nonafdo_image: |
| 185 | official_nonafdo_image = """ |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 186 | nonafdo_image { |
| 187 | chromeos_root: %s |
| 188 | build: %s |
| 189 | compiler: gcc |
| 190 | } |
| 191 | """ % (self._chromeos_root, nonafdo_image) |
Luis Lozano | 439f2b7 | 2016-01-08 11:56:02 -0800 | [diff] [blame] | 192 | f.write(official_nonafdo_image) |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 193 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 194 | label_string = '%s_trybot_image' % compiler_string |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 195 | if USE_NEXT_GCC_PATCH in self._patches: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 196 | label_string = 'gcc_next_trybot_image' |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 197 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 198 | experiment_image = """ |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 199 | %s { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 200 | chromeos_root: %s |
| 201 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 202 | compiler: %s |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 203 | } |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 204 | """ % (label_string, self._chromeos_root, trybot_image, |
| 205 | compiler_string) |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 206 | f.write(experiment_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 207 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 208 | crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 209 | noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 210 | command = ('{crosperf} --no_email=True --results_dir={r_dir} ' |
| 211 | '--json_report=True {noschedv2_opts} {exp_file}').format( |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 212 | crosperf=crosperf, |
| 213 | r_dir=self._reports_dir, |
| 214 | noschedv2_opts=noschedv2_opts, |
| 215 | exp_file=experiment_file) |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 216 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 217 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 218 | if ret != 0: |
| 219 | raise RuntimeError("Couldn't run crosperf!") |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 220 | else: |
| 221 | # Copy json report to pending archives directory. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 222 | command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 223 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 224 | return |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 225 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 226 | def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image, nonafdo_image): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 227 | """Put files in place for running seven-day reports. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 228 | |
| 229 | Create tar files of the custom and official images and copy them |
| 230 | to the weekly reports directory, so they exist when the weekly report |
| 231 | gets generated. IMPORTANT NOTE: This function must run *after* |
| 232 | crosperf has been run; otherwise the vanilla images will not be there. |
| 233 | """ |
| 234 | |
| 235 | dry_run = False |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 236 | if os.getlogin() != ROLE_ACCOUNT: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 237 | self._l.LogOutput('Running this from non-role account; not copying ' |
| 238 | 'tar files for weekly reports.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 239 | dry_run = True |
| 240 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 241 | images_path = os.path.join( |
| 242 | os.path.realpath(self._chromeos_root), 'chroot/tmp') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 243 | |
| 244 | data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board) |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 245 | dest_dir = os.path.join(data_dir, self._weekday) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 246 | if not os.path.exists(dest_dir): |
| 247 | os.makedirs(dest_dir) |
| 248 | |
| 249 | # Make sure dest_dir is empty (clean out last week's data). |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 250 | cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 251 | if dry_run: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 252 | print('CMD: %s' % cmd) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 253 | else: |
| 254 | self._ce.RunCommand(cmd) |
| 255 | |
| 256 | # Now create new tar files and copy them over. |
Luis Lozano | 439f2b7 | 2016-01-08 11:56:02 -0800 | [diff] [blame] | 257 | labels = {'test': trybot_image, 'vanilla': vanilla_image} |
| 258 | if nonafdo_image: |
| 259 | labels['nonafdo'] = nonafdo_image |
| 260 | for label_name, test_path in labels.iteritems(): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 261 | tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name) |
| 262 | cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; ' |
| 263 | 'cp %s %s/.') % (images_path, tar_file_name, test_path, |
| 264 | tar_file_name, dest_dir) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 265 | if dry_run: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 266 | print('CMD: %s' % cmd) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 267 | tar_ret = 0 |
| 268 | else: |
| 269 | tar_ret = self._ce.RunCommand(cmd) |
| 270 | if tar_ret: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 271 | self._l.LogOutput('Error while creating/copying test tar file(%s).' % |
| 272 | tar_file_name) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 273 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 274 | def _SendEmail(self): |
| 275 | """Find email message generated by crosperf and send it.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 276 | filename = os.path.join(self._reports_dir, 'msg_body.html') |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 277 | if (os.path.exists(filename) and |
| 278 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 279 | email_title = 'buildbot test results' |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 280 | if self._patches_string == USE_LLVM_PATCH: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 281 | email_title = 'buildbot llvm test results' |
| 282 | command = ('cat %s | %s -s "%s, %s" -team -html' % |
| 283 | (filename, MAIL_PROGRAM, email_title, self._board)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 284 | self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 285 | |
| 286 | def DoAll(self): |
Yunlian Jiang | 14cf596 | 2015-12-11 15:50:14 -0800 | [diff] [blame] | 287 | """Main function inside ToolchainComparator class. |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 288 | |
| 289 | Launch trybot, get image names, create crosperf experiment file, run |
| 290 | crosperf, and copy images into seven-day report directories. |
| 291 | """ |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 292 | date_str = datetime.date.today() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 293 | description = 'master_%s_%s_%s' % (self._patches_string, self._build, |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 294 | date_str) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 295 | trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, |
| 296 | self._build, |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 297 | self._patches, |
Luis Lozano | 8a68b2d | 2015-04-23 14:37:09 -0700 | [diff] [blame] | 298 | description, |
| 299 | build_toolchain=True) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 300 | |
cmtice | d54f980 | 2015-02-05 11:04:11 -0800 | [diff] [blame] | 301 | if len(trybot_image) == 0: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 302 | self._l.LogError('Unable to find trybot_image for %s!' % description) |
Luis Lozano | 7f20acb | 2015-11-04 17:15:08 -0800 | [diff] [blame] | 303 | return 1 |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 304 | |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 305 | vanilla_image = self._GetVanillaImageName(trybot_image) |
| 306 | nonafdo_image = self._GetNonAFDOImageName(trybot_image) |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 307 | |
Yunlian Jiang | 56f1376 | 2016-01-06 12:56:24 -0800 | [diff] [blame] | 308 | # The trybot image is ready here, in some cases, the vanilla image |
| 309 | # is not ready, so we need to make sure vanilla image is available. |
| 310 | buildbot_utils.WaitForImage(self._chromeos_root, vanilla_image) |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 311 | print('trybot_image: %s' % trybot_image) |
| 312 | print('vanilla_image: %s' % vanilla_image) |
| 313 | print('nonafdo_image: %s' % nonafdo_image) |
Luis Lozano | c75fd05 | 2016-02-19 17:37:01 -0800 | [diff] [blame] | 314 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 315 | if os.getlogin() == ROLE_ACCOUNT: |
| 316 | self._FinishSetup() |
| 317 | |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 318 | self._TestImages(trybot_image, vanilla_image, nonafdo_image) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 319 | self._SendEmail() |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 320 | if (self._patches_string == USE_NEXT_GCC_PATCH and |
| 321 | self._board in WEEKLY_REPORT_BOARDS): |
Luis Lozano | 7f20acb | 2015-11-04 17:15:08 -0800 | [diff] [blame] | 322 | # Only try to copy the image files if the test runs ran successfully. |
Luis Lozano | 783954f | 2015-12-21 18:06:29 -0800 | [diff] [blame] | 323 | self._CopyWeeklyReportFiles(trybot_image, vanilla_image, nonafdo_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 324 | return 0 |
| 325 | |
| 326 | |
| 327 | def Main(argv): |
| 328 | """The main function.""" |
| 329 | |
| 330 | # Common initializations |
| 331 | command_executer.InitCommandExecuter() |
| 332 | parser = optparse.OptionParser() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 333 | parser.add_option('--remote', |
| 334 | dest='remote', |
| 335 | help='Remote machines to run tests on.') |
| 336 | parser.add_option('--board', |
| 337 | dest='board', |
| 338 | default='x86-zgb', |
| 339 | help='The target board.') |
| 340 | parser.add_option('--chromeos_root', |
| 341 | dest='chromeos_root', |
| 342 | help='The chromeos root from which to run tests.') |
| 343 | parser.add_option('--weekday', |
| 344 | default='', |
| 345 | dest='weekday', |
| 346 | help='The day of the week for which to run tests.') |
| 347 | parser.add_option('--patch', |
| 348 | dest='patches', |
| 349 | help='The patches to use for the testing, ' |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 350 | "seprate the patch numbers with ',' " |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 351 | 'for more than one patches.') |
| 352 | parser.add_option('--noschedv2', |
| 353 | dest='noschedv2', |
| 354 | action='store_true', |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 355 | default=False, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 356 | help='Pass --noschedv2 to crosperf.') |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 357 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 358 | options, _ = parser.parse_args(argv) |
| 359 | if not options.board: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 360 | print('Please give a board.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 361 | return 1 |
| 362 | if not options.remote: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 363 | print('Please give at least one remote machine.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 364 | return 1 |
| 365 | if not options.chromeos_root: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 366 | print('Please specify the ChromeOS root directory.') |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 367 | return 1 |
Yunlian Jiang | 76259e6 | 2015-08-21 08:44:31 -0700 | [diff] [blame] | 368 | if options.patches: |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 369 | patches = options.patches |
| 370 | else: |
| 371 | patches = USE_NEXT_GCC_PATCH |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 372 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 373 | fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, |
| 374 | options.weekday, patches, options.noschedv2) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 375 | return fc.DoAll() |
| 376 | |
| 377 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 378 | if __name__ == '__main__': |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 379 | retval = Main(sys.argv) |
| 380 | sys.exit(retval) |