cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | """ |
| 3 | Script for running nightly compiler tests on ChromeOS. |
| 4 | |
| 5 | This script launches a buildbot to build ChromeOS with the latest compiler on |
| 6 | a particular board; then it finds and downloads the trybot image and the |
| 7 | corresponding official image, and runs crosperf performance tests comparing |
| 8 | the two. It then generates a report, emails it to the c-compiler-chrome, as |
| 9 | well as copying the images into the seven-day reports directory. |
| 10 | """ |
| 11 | |
| 12 | # Script to test different toolchains against ChromeOS benchmarks. |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 13 | import datetime |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 14 | import optparse |
| 15 | import os |
| 16 | import sys |
| 17 | import time |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 18 | |
| 19 | from utils import command_executer |
| 20 | from utils import logger |
| 21 | |
| 22 | from utils import buildbot_utils |
| 23 | |
| 24 | # CL that updated GCC ebuilds to use 'next_gcc'. |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 25 | USE_NEXT_GCC_PATCH = "230260" |
| 26 | |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 27 | # CL that uses LLVM to build the peppy image. |
| 28 | USE_LLVM_PATCH = "295217" |
| 29 | |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 30 | # The boards on which we run weekly reports |
| 31 | WEEKLY_REPORT_BOARDS = ["lumpy"] |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 32 | |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 33 | CROSTC_ROOT = "/usr/local/google/crostc" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 34 | ROLE_ACCOUNT = "mobiletc-prebuild" |
| 35 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 36 | MAIL_PROGRAM = "~/var/bin/mail-sheriff" |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 37 | WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data") |
| 38 | PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives") |
| 39 | NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports") |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 40 | |
| 41 | class ToolchainComparator(): |
| 42 | """ |
| 43 | Class for doing the nightly tests work. |
| 44 | """ |
| 45 | |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 46 | def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False): |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 47 | self._board = board |
| 48 | self._remotes = remotes |
| 49 | self._chromeos_root = chromeos_root |
| 50 | self._base_dir = os.getcwd() |
| 51 | self._ce = command_executer.GetCommandExecuter() |
| 52 | self._l = logger.GetLogger() |
| 53 | self._build = "%s-release" % board |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 54 | self._patches = patches.split(',') |
| 55 | self._patches_string = '_'.join(str(p) for p in self._patches) |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 56 | self._noschedv2 = noschedv2 |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 57 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 58 | if not weekday: |
| 59 | self._weekday = time.strftime("%a") |
| 60 | else: |
| 61 | self._weekday = weekday |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 62 | timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
| 63 | "%Y-%m-%d_%H:%M:%S") |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 64 | self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR, |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 65 | "%s.%s" % (timestamp, board), |
| 66 | ) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 67 | |
| 68 | def _ParseVanillaImage(self, trybot_image): |
| 69 | """ |
| 70 | Parse a trybot artifact name to get corresponding vanilla image. |
| 71 | |
| 72 | This function takes an artifact name, such as |
| 73 | 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the |
| 74 | corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'. |
| 75 | """ |
| 76 | start_pos = trybot_image.find(self._build) |
| 77 | end_pos = trybot_image.rfind("-b") |
| 78 | vanilla_image = trybot_image[start_pos:end_pos] |
| 79 | return vanilla_image |
| 80 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 81 | def _FinishSetup(self): |
| 82 | """ |
| 83 | Make sure testing_rsa file is properly set up. |
| 84 | """ |
| 85 | # Fix protections on ssh key |
| 86 | command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target" |
| 87 | "/chrome-src-internal/src/third_party/chromite/ssh_keys" |
| 88 | "/testing_rsa") |
| 89 | ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 90 | if ret_val != 0: |
| 91 | raise RuntimeError("chmod for testing_rsa failed") |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 92 | |
| 93 | def _TestImages(self, trybot_image, vanilla_image): |
| 94 | """ |
| 95 | Create crosperf experiment file. |
| 96 | |
| 97 | Given the names of the trybot and vanilla images, create the |
| 98 | appropriate crosperf experiment file and launch crosperf on it. |
| 99 | """ |
| 100 | experiment_file_dir = os.path.join (self._chromeos_root, "..", |
| 101 | self._weekday) |
| 102 | experiment_file_name = "%s_toolchain_experiment.txt" % self._board |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 103 | |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 104 | compiler_string = "gcc" |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 105 | if self._patches_string == USE_LLVM_PATCH: |
| 106 | experiment_file_name = "%s_llvm_experiment.txt" % self._board |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 107 | compiler_string = "llvm" |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 108 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 109 | experiment_file = os.path.join (experiment_file_dir, |
| 110 | experiment_file_name) |
| 111 | experiment_header = """ |
| 112 | board: %s |
| 113 | remote: %s |
Luis Lozano | e1efeb8 | 2015-06-16 16:35:44 -0700 | [diff] [blame] | 114 | retries: 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 115 | """ % (self._board, self._remotes) |
| 116 | experiment_tests = """ |
Luis Lozano | 1489d64 | 2015-12-08 10:08:19 -0800 | [diff] [blame] | 117 | benchmark: all_toolchain_perf { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 118 | suite: telemetry_Crosperf |
| 119 | iterations: 3 |
| 120 | } |
| 121 | """ |
| 122 | with open(experiment_file, "w") as f: |
| 123 | print >> f, experiment_header |
| 124 | print >> f, experiment_tests |
| 125 | |
| 126 | # Now add vanilla to test file. |
| 127 | official_image = """ |
| 128 | vanilla_image { |
| 129 | chromeos_root: %s |
| 130 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 131 | compiler: gcc |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 132 | } |
| 133 | """ % (self._chromeos_root, vanilla_image) |
| 134 | print >> f, official_image |
| 135 | |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 136 | label_string = "%s_trybot_image" % compiler_string |
| 137 | if USE_NEXT_GCC_PATCH in self._patches: |
| 138 | label_string = "gcc_next_trybot_image" |
| 139 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 140 | experiment_image = """ |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 141 | %s { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 142 | chromeos_root: %s |
| 143 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 144 | compiler: %s |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 145 | } |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 146 | """ % (label_string, self._chromeos_root, trybot_image, |
| 147 | compiler_string) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 148 | print >> f, experiment_image |
| 149 | |
| 150 | crosperf = os.path.join(TOOLCHAIN_DIR, |
| 151 | "crosperf", |
| 152 | "crosperf") |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 153 | noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 154 | command = ("{crosperf} --no_email=True --results_dir={r_dir} " |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 155 | "--json_report=True {noschedv2_opts} {exp_file}").format( |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 156 | crosperf=crosperf, |
| 157 | r_dir=self._reports_dir, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 158 | noschedv2_opts=noschedv2_opts, |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 159 | exp_file=experiment_file) |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 160 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 161 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 162 | if ret != 0: |
| 163 | raise RuntimeError("Couldn't run crosperf!") |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 164 | else: |
| 165 | # Copy json report to pending archives directory. |
| 166 | command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR) |
| 167 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 168 | return |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 169 | |
| 170 | def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image): |
| 171 | """ |
| 172 | Put files in place for running seven-day reports. |
| 173 | |
| 174 | Create tar files of the custom and official images and copy them |
| 175 | to the weekly reports directory, so they exist when the weekly report |
| 176 | gets generated. IMPORTANT NOTE: This function must run *after* |
| 177 | crosperf has been run; otherwise the vanilla images will not be there. |
| 178 | """ |
| 179 | |
| 180 | dry_run = False |
| 181 | if (os.getlogin() != ROLE_ACCOUNT): |
| 182 | self._l.LogOutput("Running this from non-role account; not copying " |
| 183 | "tar files for weekly reports.") |
| 184 | dry_run = True |
| 185 | |
| 186 | images_path = os.path.join(os.path.realpath(self._chromeos_root), |
| 187 | "chroot/tmp") |
| 188 | |
| 189 | data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board) |
| 190 | dest_dir = os.path.join (data_dir, self._weekday) |
| 191 | if not os.path.exists(dest_dir): |
| 192 | os.makedirs(dest_dir) |
| 193 | |
| 194 | # Make sure dest_dir is empty (clean out last week's data). |
| 195 | cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday) |
| 196 | if dry_run: |
| 197 | print "CMD: %s" % cmd |
| 198 | else: |
| 199 | self._ce.RunCommand(cmd) |
| 200 | |
| 201 | # Now create new tar files and copy them over. |
| 202 | labels = [ "test", "vanilla" ] |
| 203 | for label_name in labels: |
| 204 | if label_name == "test": |
| 205 | test_path = trybot_image |
| 206 | else: |
| 207 | test_path = vanilla_image |
| 208 | tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name) |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 209 | cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; " |
| 210 | "cp %s %s/.") % (images_path, |
| 211 | tar_file_name, |
| 212 | test_path, |
| 213 | tar_file_name, |
| 214 | dest_dir) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 215 | if dry_run: |
| 216 | print "CMD: %s" % cmd |
| 217 | tar_ret = 0 |
| 218 | else: |
| 219 | tar_ret = self._ce.RunCommand(cmd) |
| 220 | if tar_ret: |
| 221 | self._l.LogOutput("Error while creating/copying test tar file(%s)." |
| 222 | % tar_file_name) |
| 223 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 224 | def _SendEmail(self): |
| 225 | """Find email message generated by crosperf and send it.""" |
| 226 | filename = os.path.join(self._reports_dir, |
| 227 | "msg_body.html") |
| 228 | if (os.path.exists(filename) and |
| 229 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 230 | email_title = "buildbot test results" |
| 231 | if self._patches_string == USE_LLVM_PATCH: |
| 232 | email_title = "buildbot llvm test results" |
| 233 | command = ('cat %s | %s -s "%s, %s" -team -html' |
| 234 | % (filename, MAIL_PROGRAM, email_title, self._board)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 235 | self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 236 | |
| 237 | def DoAll(self): |
| 238 | """ |
| 239 | Main function inside ToolchainComparator class. |
| 240 | |
| 241 | Launch trybot, get image names, create crosperf experiment file, run |
| 242 | crosperf, and copy images into seven-day report directories. |
| 243 | """ |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 244 | date_str = datetime.date.today() |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 245 | description = "master_%s_%s_%s" % (self._patches_string, |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 246 | self._build, |
| 247 | date_str) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 248 | trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, |
| 249 | self._build, |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 250 | self._patches, |
Luis Lozano | 8a68b2d | 2015-04-23 14:37:09 -0700 | [diff] [blame] | 251 | description, |
| 252 | build_toolchain=True) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 253 | |
| 254 | vanilla_image = self._ParseVanillaImage(trybot_image) |
| 255 | |
| 256 | print ("trybot_image: %s" % trybot_image) |
| 257 | print ("vanilla_image: %s" % vanilla_image) |
cmtice | d54f980 | 2015-02-05 11:04:11 -0800 | [diff] [blame] | 258 | if len(trybot_image) == 0: |
Luis Lozano | 7f20acb | 2015-11-04 17:15:08 -0800 | [diff] [blame] | 259 | self._l.LogError("Unable to find trybot_image for %s!" % description) |
| 260 | return 1 |
cmtice | d54f980 | 2015-02-05 11:04:11 -0800 | [diff] [blame] | 261 | if len(vanilla_image) == 0: |
Luis Lozano | 7f20acb | 2015-11-04 17:15:08 -0800 | [diff] [blame] | 262 | self._l.LogError("Unable to find vanilla image for %s!" % description) |
| 263 | return 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 264 | if os.getlogin() == ROLE_ACCOUNT: |
| 265 | self._FinishSetup() |
| 266 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 267 | self._TestImages(trybot_image, vanilla_image) |
| 268 | self._SendEmail() |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 269 | if (self._patches_string == USE_NEXT_GCC_PATCH and |
| 270 | self._board in WEEKLY_REPORT_BOARDS): |
Luis Lozano | 7f20acb | 2015-11-04 17:15:08 -0800 | [diff] [blame] | 271 | # Only try to copy the image files if the test runs ran successfully. |
| 272 | self._CopyWeeklyReportFiles(trybot_image, vanilla_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 273 | return 0 |
| 274 | |
| 275 | |
| 276 | def Main(argv): |
| 277 | """The main function.""" |
| 278 | |
| 279 | # Common initializations |
| 280 | command_executer.InitCommandExecuter() |
| 281 | parser = optparse.OptionParser() |
| 282 | parser.add_option("--remote", |
| 283 | dest="remote", |
| 284 | help="Remote machines to run tests on.") |
| 285 | parser.add_option("--board", |
| 286 | dest="board", |
| 287 | default="x86-zgb", |
| 288 | help="The target board.") |
| 289 | parser.add_option("--chromeos_root", |
| 290 | dest="chromeos_root", |
| 291 | help="The chromeos root from which to run tests.") |
| 292 | parser.add_option("--weekday", default="", |
| 293 | dest="weekday", |
| 294 | help="The day of the week for which to run tests.") |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 295 | parser.add_option("--patch", |
| 296 | dest="patches", |
| 297 | help="The patches to use for the testing, " |
| 298 | "seprate the patch numbers with ',' " |
| 299 | "for more than one patches.") |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 300 | parser.add_option("--noschedv2", |
| 301 | dest="noschedv2", |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 302 | action="store_true", |
| 303 | default=False, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 304 | help="Pass --noschedv2 to crosperf.") |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 305 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 306 | options, _ = parser.parse_args(argv) |
| 307 | if not options.board: |
| 308 | print "Please give a board." |
| 309 | return 1 |
| 310 | if not options.remote: |
| 311 | print "Please give at least one remote machine." |
| 312 | return 1 |
| 313 | if not options.chromeos_root: |
| 314 | print "Please specify the ChromeOS root directory." |
| 315 | return 1 |
Yunlian Jiang | 76259e6 | 2015-08-21 08:44:31 -0700 | [diff] [blame] | 316 | if options.patches: |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 317 | patches = options.patches |
| 318 | else: |
| 319 | patches = USE_NEXT_GCC_PATCH |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 320 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 321 | fc = ToolchainComparator(options.board, options.remote, |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 322 | options.chromeos_root, options.weekday, patches, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 323 | options.noschedv2) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 324 | return fc.DoAll() |
| 325 | |
| 326 | |
| 327 | if __name__ == "__main__": |
| 328 | retval = Main(sys.argv) |
| 329 | sys.exit(retval) |