cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | """ |
| 3 | Script for running nightly compiler tests on ChromeOS. |
| 4 | |
| 5 | This script launches a buildbot to build ChromeOS with the latest compiler on |
| 6 | a particular board; then it finds and downloads the trybot image and the |
| 7 | corresponding official image, and runs crosperf performance tests comparing |
| 8 | the two. It then generates a report, emails it to the c-compiler-chrome, as |
| 9 | well as copying the images into the seven-day reports directory. |
| 10 | """ |
| 11 | |
| 12 | # Script to test different toolchains against ChromeOS benchmarks. |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 13 | import datetime |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 14 | import optparse |
| 15 | import os |
| 16 | import sys |
| 17 | import time |
| 18 | import urllib2 |
| 19 | |
| 20 | from utils import command_executer |
| 21 | from utils import logger |
| 22 | |
| 23 | from utils import buildbot_utils |
| 24 | |
| 25 | # CL that updated GCC ebuilds to use 'next_gcc'. |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 26 | USE_NEXT_GCC_PATCH = "230260" |
| 27 | |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 28 | # CL that uses LLVM to build the peppy image. |
| 29 | USE_LLVM_PATCH = "295217" |
| 30 | |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 31 | # The boards on which we run weekly reports |
| 32 | WEEKLY_REPORT_BOARDS = ["lumpy"] |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 33 | |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 34 | CROSTC_ROOT = "/usr/local/google/crostc" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 35 | ROLE_ACCOUNT = "mobiletc-prebuild" |
| 36 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 37 | MAIL_PROGRAM = "~/var/bin/mail-sheriff" |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 38 | WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data") |
| 39 | PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives") |
| 40 | NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports") |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 41 | |
| 42 | class ToolchainComparator(): |
| 43 | """ |
| 44 | Class for doing the nightly tests work. |
| 45 | """ |
| 46 | |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 47 | def __init__(self, board, remotes, chromeos_root, weekday, patches, noschedv2=False): |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 48 | self._board = board |
| 49 | self._remotes = remotes |
| 50 | self._chromeos_root = chromeos_root |
| 51 | self._base_dir = os.getcwd() |
| 52 | self._ce = command_executer.GetCommandExecuter() |
| 53 | self._l = logger.GetLogger() |
| 54 | self._build = "%s-release" % board |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 55 | self._patches = patches.split(',') |
| 56 | self._patches_string = '_'.join(str(p) for p in self._patches) |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 57 | self._noschedv2 = noschedv2 |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 58 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 59 | if not weekday: |
| 60 | self._weekday = time.strftime("%a") |
| 61 | else: |
| 62 | self._weekday = weekday |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 63 | timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
| 64 | "%Y-%m-%d_%H:%M:%S") |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 65 | self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR, |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 66 | "%s.%s" % (timestamp, board), |
| 67 | ) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 68 | |
| 69 | def _ParseVanillaImage(self, trybot_image): |
| 70 | """ |
| 71 | Parse a trybot artifact name to get corresponding vanilla image. |
| 72 | |
| 73 | This function takes an artifact name, such as |
| 74 | 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the |
| 75 | corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'. |
| 76 | """ |
| 77 | start_pos = trybot_image.find(self._build) |
| 78 | end_pos = trybot_image.rfind("-b") |
| 79 | vanilla_image = trybot_image[start_pos:end_pos] |
| 80 | return vanilla_image |
| 81 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 82 | def _FinishSetup(self): |
| 83 | """ |
| 84 | Make sure testing_rsa file is properly set up. |
| 85 | """ |
| 86 | # Fix protections on ssh key |
| 87 | command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target" |
| 88 | "/chrome-src-internal/src/third_party/chromite/ssh_keys" |
| 89 | "/testing_rsa") |
| 90 | ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 91 | if ret_val != 0: |
| 92 | raise RuntimeError("chmod for testing_rsa failed") |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 93 | |
| 94 | def _TestImages(self, trybot_image, vanilla_image): |
| 95 | """ |
| 96 | Create crosperf experiment file. |
| 97 | |
| 98 | Given the names of the trybot and vanilla images, create the |
| 99 | appropriate crosperf experiment file and launch crosperf on it. |
| 100 | """ |
| 101 | experiment_file_dir = os.path.join (self._chromeos_root, "..", |
| 102 | self._weekday) |
| 103 | experiment_file_name = "%s_toolchain_experiment.txt" % self._board |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 104 | |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 105 | compiler_string = "gcc" |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 106 | if self._patches_string == USE_LLVM_PATCH: |
| 107 | experiment_file_name = "%s_llvm_experiment.txt" % self._board |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 108 | compiler_string = "llvm" |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 109 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 110 | experiment_file = os.path.join (experiment_file_dir, |
| 111 | experiment_file_name) |
| 112 | experiment_header = """ |
| 113 | board: %s |
| 114 | remote: %s |
Luis Lozano | e1efeb8 | 2015-06-16 16:35:44 -0700 | [diff] [blame] | 115 | retries: 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 116 | """ % (self._board, self._remotes) |
| 117 | experiment_tests = """ |
cmtice | 0c84ea7 | 2015-06-25 14:22:36 -0700 | [diff] [blame] | 118 | benchmark: all_toolchain_perf { |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 119 | suite: telemetry_Crosperf |
| 120 | iterations: 3 |
| 121 | } |
| 122 | """ |
| 123 | with open(experiment_file, "w") as f: |
| 124 | print >> f, experiment_header |
| 125 | print >> f, experiment_tests |
| 126 | |
| 127 | # Now add vanilla to test file. |
| 128 | official_image = """ |
| 129 | vanilla_image { |
| 130 | chromeos_root: %s |
| 131 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 132 | compiler: gcc |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 133 | } |
| 134 | """ % (self._chromeos_root, vanilla_image) |
| 135 | print >> f, official_image |
| 136 | |
| 137 | experiment_image = """ |
| 138 | test_image { |
| 139 | chromeos_root: %s |
| 140 | build: %s |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 141 | compiler: %s |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 142 | } |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 143 | """ % (self._chromeos_root, trybot_image, compiler_string) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 144 | print >> f, experiment_image |
| 145 | |
| 146 | crosperf = os.path.join(TOOLCHAIN_DIR, |
| 147 | "crosperf", |
| 148 | "crosperf") |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 149 | noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 150 | command = ("{crosperf} --no_email=True --results_dir={r_dir} " |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 151 | "--json_report=True {noschedv2_opts} {exp_file}").format( |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 152 | crosperf=crosperf, |
| 153 | r_dir=self._reports_dir, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 154 | noschedv2_opts=noschedv2_opts, |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 155 | exp_file=experiment_file) |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 156 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 157 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 158 | if ret != 0: |
| 159 | raise RuntimeError("Couldn't run crosperf!") |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 160 | else: |
| 161 | # Copy json report to pending archives directory. |
| 162 | command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR) |
| 163 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 164 | return |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 165 | |
| 166 | def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image): |
| 167 | """ |
| 168 | Put files in place for running seven-day reports. |
| 169 | |
| 170 | Create tar files of the custom and official images and copy them |
| 171 | to the weekly reports directory, so they exist when the weekly report |
| 172 | gets generated. IMPORTANT NOTE: This function must run *after* |
| 173 | crosperf has been run; otherwise the vanilla images will not be there. |
| 174 | """ |
| 175 | |
| 176 | dry_run = False |
| 177 | if (os.getlogin() != ROLE_ACCOUNT): |
| 178 | self._l.LogOutput("Running this from non-role account; not copying " |
| 179 | "tar files for weekly reports.") |
| 180 | dry_run = True |
| 181 | |
| 182 | images_path = os.path.join(os.path.realpath(self._chromeos_root), |
| 183 | "chroot/tmp") |
| 184 | |
| 185 | data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board) |
| 186 | dest_dir = os.path.join (data_dir, self._weekday) |
| 187 | if not os.path.exists(dest_dir): |
| 188 | os.makedirs(dest_dir) |
| 189 | |
| 190 | # Make sure dest_dir is empty (clean out last week's data). |
| 191 | cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday) |
| 192 | if dry_run: |
| 193 | print "CMD: %s" % cmd |
| 194 | else: |
| 195 | self._ce.RunCommand(cmd) |
| 196 | |
| 197 | # Now create new tar files and copy them over. |
| 198 | labels = [ "test", "vanilla" ] |
| 199 | for label_name in labels: |
| 200 | if label_name == "test": |
| 201 | test_path = trybot_image |
| 202 | else: |
| 203 | test_path = vanilla_image |
| 204 | tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name) |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 205 | cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; " |
| 206 | "cp %s %s/.") % (images_path, |
| 207 | tar_file_name, |
| 208 | test_path, |
| 209 | tar_file_name, |
| 210 | dest_dir) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 211 | if dry_run: |
| 212 | print "CMD: %s" % cmd |
| 213 | tar_ret = 0 |
| 214 | else: |
| 215 | tar_ret = self._ce.RunCommand(cmd) |
| 216 | if tar_ret: |
| 217 | self._l.LogOutput("Error while creating/copying test tar file(%s)." |
| 218 | % tar_file_name) |
| 219 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 220 | def _SendEmail(self): |
| 221 | """Find email message generated by crosperf and send it.""" |
| 222 | filename = os.path.join(self._reports_dir, |
| 223 | "msg_body.html") |
| 224 | if (os.path.exists(filename) and |
| 225 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
Yunlian Jiang | 2f56356 | 2015-08-28 13:54:04 -0700 | [diff] [blame] | 226 | email_title = "buildbot test results" |
| 227 | if self._patches_string == USE_LLVM_PATCH: |
| 228 | email_title = "buildbot llvm test results" |
| 229 | command = ('cat %s | %s -s "%s, %s" -team -html' |
| 230 | % (filename, MAIL_PROGRAM, email_title, self._board)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 231 | self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 232 | |
| 233 | def DoAll(self): |
| 234 | """ |
| 235 | Main function inside ToolchainComparator class. |
| 236 | |
| 237 | Launch trybot, get image names, create crosperf experiment file, run |
| 238 | crosperf, and copy images into seven-day report directories. |
| 239 | """ |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 240 | date_str = datetime.date.today() |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 241 | description = "master_%s_%s_%s" % (self._patches_string, |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 242 | self._build, |
| 243 | date_str) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 244 | trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, |
| 245 | self._build, |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 246 | self._patches, |
Luis Lozano | 8a68b2d | 2015-04-23 14:37:09 -0700 | [diff] [blame] | 247 | description, |
| 248 | build_toolchain=True) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 249 | |
| 250 | vanilla_image = self._ParseVanillaImage(trybot_image) |
| 251 | |
| 252 | print ("trybot_image: %s" % trybot_image) |
| 253 | print ("vanilla_image: %s" % vanilla_image) |
cmtice | d54f980 | 2015-02-05 11:04:11 -0800 | [diff] [blame] | 254 | if len(trybot_image) == 0: |
| 255 | self._l.LogError("Unable to find trybot_image for %s!" % description) |
| 256 | return 1 |
| 257 | if len(vanilla_image) == 0: |
| 258 | self._l.LogError("Unable to find vanilla image for %s!" % description) |
| 259 | return 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 260 | if os.getlogin() == ROLE_ACCOUNT: |
| 261 | self._FinishSetup() |
| 262 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 263 | self._TestImages(trybot_image, vanilla_image) |
| 264 | self._SendEmail() |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 265 | if (self._patches_string == USE_NEXT_GCC_PATCH and |
| 266 | self._board in WEEKLY_REPORT_BOARDS): |
| 267 | # Only try to copy the image files if the test runs ran successfully. |
| 268 | self._CopyWeeklyReportFiles(trybot_image, vanilla_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 269 | return 0 |
| 270 | |
| 271 | |
| 272 | def Main(argv): |
| 273 | """The main function.""" |
| 274 | |
| 275 | # Common initializations |
| 276 | command_executer.InitCommandExecuter() |
| 277 | parser = optparse.OptionParser() |
| 278 | parser.add_option("--remote", |
| 279 | dest="remote", |
| 280 | help="Remote machines to run tests on.") |
| 281 | parser.add_option("--board", |
| 282 | dest="board", |
| 283 | default="x86-zgb", |
| 284 | help="The target board.") |
| 285 | parser.add_option("--chromeos_root", |
| 286 | dest="chromeos_root", |
| 287 | help="The chromeos root from which to run tests.") |
| 288 | parser.add_option("--weekday", default="", |
| 289 | dest="weekday", |
| 290 | help="The day of the week for which to run tests.") |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 291 | parser.add_option("--patch", |
| 292 | dest="patches", |
| 293 | help="The patches to use for the testing, " |
| 294 | "seprate the patch numbers with ',' " |
| 295 | "for more than one patches.") |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 296 | parser.add_option("--noschedv2", |
| 297 | dest="noschedv2", |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 298 | action="store_true", |
| 299 | default=False, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 300 | help="Pass --noschedv2 to crosperf.") |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 301 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 302 | options, _ = parser.parse_args(argv) |
| 303 | if not options.board: |
| 304 | print "Please give a board." |
| 305 | return 1 |
| 306 | if not options.remote: |
| 307 | print "Please give at least one remote machine." |
| 308 | return 1 |
| 309 | if not options.chromeos_root: |
| 310 | print "Please specify the ChromeOS root directory." |
| 311 | return 1 |
Yunlian Jiang | 76259e6 | 2015-08-21 08:44:31 -0700 | [diff] [blame] | 312 | if options.patches: |
Yunlian Jiang | 3c6e467 | 2015-08-24 15:58:22 -0700 | [diff] [blame] | 313 | patches = options.patches |
| 314 | else: |
| 315 | patches = USE_NEXT_GCC_PATCH |
Yunlian Jiang | e52838c | 2015-08-20 14:32:37 -0700 | [diff] [blame] | 316 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 317 | fc = ToolchainComparator(options.board, options.remote, |
Han Shen | 3641312 | 2015-08-28 11:05:40 -0700 | [diff] [blame] | 318 | options.chromeos_root, options.weekday, patches, |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 319 | options.noschedv2) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 320 | return fc.DoAll() |
| 321 | |
| 322 | |
| 323 | if __name__ == "__main__": |
| 324 | retval = Main(sys.argv) |
| 325 | sys.exit(retval) |