cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | """ |
| 3 | Script for running nightly compiler tests on ChromeOS. |
| 4 | |
| 5 | This script launches a buildbot to build ChromeOS with the latest compiler on |
| 6 | a particular board; then it finds and downloads the trybot image and the |
| 7 | corresponding official image, and runs crosperf performance tests comparing |
| 8 | the two. It then generates a report, emails it to the c-compiler-chrome, as |
| 9 | well as copying the images into the seven-day reports directory. |
| 10 | """ |
| 11 | |
| 12 | # Script to test different toolchains against ChromeOS benchmarks. |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 13 | import datetime |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 14 | import optparse |
| 15 | import os |
| 16 | import sys |
| 17 | import time |
| 18 | import urllib2 |
| 19 | |
| 20 | from utils import command_executer |
| 21 | from utils import logger |
| 22 | |
| 23 | from utils import buildbot_utils |
| 24 | |
| 25 | # CL that updated GCC ebuilds to use 'next_gcc'. |
| 26 | USE_NEXT_GCC_PATCH ="230260" |
| 27 | |
| 28 | WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data" |
| 29 | ROLE_ACCOUNT = "mobiletc-prebuild" |
| 30 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 31 | MAIL_PROGRAM = "~/var/bin/mail-sheriff" |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 32 | |
| 33 | class ToolchainComparator(): |
| 34 | """ |
| 35 | Class for doing the nightly tests work. |
| 36 | """ |
| 37 | |
| 38 | def __init__(self, board, remotes, chromeos_root, weekday): |
| 39 | self._board = board |
| 40 | self._remotes = remotes |
| 41 | self._chromeos_root = chromeos_root |
| 42 | self._base_dir = os.getcwd() |
| 43 | self._ce = command_executer.GetCommandExecuter() |
| 44 | self._l = logger.GetLogger() |
| 45 | self._build = "%s-release" % board |
| 46 | if not weekday: |
| 47 | self._weekday = time.strftime("%a") |
| 48 | else: |
| 49 | self._weekday = weekday |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 50 | timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
| 51 | "%Y-%m-%d_%H:%M:%S") |
| 52 | self._reports_dir = os.path.join( |
| 53 | os.path.expanduser("~/nightly_test_reports"), |
| 54 | "%s.%s" % (timestamp, board), |
| 55 | ) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 56 | |
| 57 | def _ParseVanillaImage(self, trybot_image): |
| 58 | """ |
| 59 | Parse a trybot artifact name to get corresponding vanilla image. |
| 60 | |
| 61 | This function takes an artifact name, such as |
| 62 | 'trybot-daisy-release/R40-6394.0.0-b1389', and returns the |
| 63 | corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'. |
| 64 | """ |
| 65 | start_pos = trybot_image.find(self._build) |
| 66 | end_pos = trybot_image.rfind("-b") |
| 67 | vanilla_image = trybot_image[start_pos:end_pos] |
| 68 | return vanilla_image |
| 69 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 70 | def _FinishSetup(self): |
| 71 | """ |
| 72 | Make sure testing_rsa file is properly set up. |
| 73 | """ |
| 74 | # Fix protections on ssh key |
| 75 | command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target" |
| 76 | "/chrome-src-internal/src/third_party/chromite/ssh_keys" |
| 77 | "/testing_rsa") |
| 78 | ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 79 | if ret_val != 0: |
| 80 | raise RuntimeError("chmod for testing_rsa failed") |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 81 | |
| 82 | def _TestImages(self, trybot_image, vanilla_image): |
| 83 | """ |
| 84 | Create crosperf experiment file. |
| 85 | |
| 86 | Given the names of the trybot and vanilla images, create the |
| 87 | appropriate crosperf experiment file and launch crosperf on it. |
| 88 | """ |
| 89 | experiment_file_dir = os.path.join (self._chromeos_root, "..", |
| 90 | self._weekday) |
| 91 | experiment_file_name = "%s_toolchain_experiment.txt" % self._board |
| 92 | experiment_file = os.path.join (experiment_file_dir, |
| 93 | experiment_file_name) |
| 94 | experiment_header = """ |
| 95 | board: %s |
| 96 | remote: %s |
Luis Lozano | e1efeb8 | 2015-06-16 16:35:44 -0700 | [diff] [blame] | 97 | retries: 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 98 | """ % (self._board, self._remotes) |
| 99 | experiment_tests = """ |
| 100 | benchmark: all_perfv2 { |
| 101 | suite: telemetry_Crosperf |
| 102 | iterations: 3 |
| 103 | } |
| 104 | """ |
| 105 | with open(experiment_file, "w") as f: |
| 106 | print >> f, experiment_header |
| 107 | print >> f, experiment_tests |
| 108 | |
| 109 | # Now add vanilla to test file. |
| 110 | official_image = """ |
| 111 | vanilla_image { |
| 112 | chromeos_root: %s |
| 113 | build: %s |
| 114 | } |
| 115 | """ % (self._chromeos_root, vanilla_image) |
| 116 | print >> f, official_image |
| 117 | |
| 118 | experiment_image = """ |
| 119 | test_image { |
| 120 | chromeos_root: %s |
| 121 | build: %s |
| 122 | } |
| 123 | """ % (self._chromeos_root, trybot_image) |
| 124 | print >> f, experiment_image |
| 125 | |
| 126 | crosperf = os.path.join(TOOLCHAIN_DIR, |
| 127 | "crosperf", |
| 128 | "crosperf") |
cmtice | 9db7ae5 | 2015-06-23 10:34:39 -0700 | [diff] [blame] | 129 | command = ("%s --no_email=True --results_dir=%s %s" % |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 130 | (crosperf, self._reports_dir, experiment_file)) |
| 131 | |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 132 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 133 | if ret != 0: |
| 134 | raise RuntimeError("Couldn't run crosperf!") |
| 135 | return |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 136 | |
| 137 | def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image): |
| 138 | """ |
| 139 | Put files in place for running seven-day reports. |
| 140 | |
| 141 | Create tar files of the custom and official images and copy them |
| 142 | to the weekly reports directory, so they exist when the weekly report |
| 143 | gets generated. IMPORTANT NOTE: This function must run *after* |
| 144 | crosperf has been run; otherwise the vanilla images will not be there. |
| 145 | """ |
| 146 | |
| 147 | dry_run = False |
| 148 | if (os.getlogin() != ROLE_ACCOUNT): |
| 149 | self._l.LogOutput("Running this from non-role account; not copying " |
| 150 | "tar files for weekly reports.") |
| 151 | dry_run = True |
| 152 | |
| 153 | images_path = os.path.join(os.path.realpath(self._chromeos_root), |
| 154 | "chroot/tmp") |
| 155 | |
| 156 | data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board) |
| 157 | dest_dir = os.path.join (data_dir, self._weekday) |
| 158 | if not os.path.exists(dest_dir): |
| 159 | os.makedirs(dest_dir) |
| 160 | |
| 161 | # Make sure dest_dir is empty (clean out last week's data). |
| 162 | cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday) |
| 163 | if dry_run: |
| 164 | print "CMD: %s" % cmd |
| 165 | else: |
| 166 | self._ce.RunCommand(cmd) |
| 167 | |
| 168 | # Now create new tar files and copy them over. |
| 169 | labels = [ "test", "vanilla" ] |
| 170 | for label_name in labels: |
| 171 | if label_name == "test": |
| 172 | test_path = trybot_image |
| 173 | else: |
| 174 | test_path = vanilla_image |
| 175 | tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name) |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 176 | cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; " |
| 177 | "cp %s %s/.") % (images_path, |
| 178 | tar_file_name, |
| 179 | test_path, |
| 180 | tar_file_name, |
| 181 | dest_dir) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 182 | if dry_run: |
| 183 | print "CMD: %s" % cmd |
| 184 | tar_ret = 0 |
| 185 | else: |
| 186 | tar_ret = self._ce.RunCommand(cmd) |
| 187 | if tar_ret: |
| 188 | self._l.LogOutput("Error while creating/copying test tar file(%s)." |
| 189 | % tar_file_name) |
| 190 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 191 | def _SendEmail(self): |
| 192 | """Find email message generated by crosperf and send it.""" |
| 193 | filename = os.path.join(self._reports_dir, |
| 194 | "msg_body.html") |
| 195 | if (os.path.exists(filename) and |
| 196 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
| 197 | command = ('cat %s | %s -s "buildbot test results, %s" -team -html' |
| 198 | % (filename, MAIL_PROGRAM, self._board)) |
| 199 | self._ce.RunCommand(command) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 200 | |
| 201 | def DoAll(self): |
| 202 | """ |
| 203 | Main function inside ToolchainComparator class. |
| 204 | |
| 205 | Launch trybot, get image names, create crosperf experiment file, run |
| 206 | crosperf, and copy images into seven-day report directories. |
| 207 | """ |
cmtice | ce5ffa4 | 2015-02-12 15:18:43 -0800 | [diff] [blame] | 208 | date_str = datetime.date.today() |
Han Shen | fe054f1 | 2015-02-18 15:00:13 -0800 | [diff] [blame] | 209 | description = "master_%s_%s_%s" % (USE_NEXT_GCC_PATCH, |
| 210 | self._build, |
| 211 | date_str) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 212 | trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root, |
| 213 | self._build, |
| 214 | [ USE_NEXT_GCC_PATCH ], |
Luis Lozano | 8a68b2d | 2015-04-23 14:37:09 -0700 | [diff] [blame] | 215 | description, |
| 216 | build_toolchain=True) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 217 | |
| 218 | vanilla_image = self._ParseVanillaImage(trybot_image) |
| 219 | |
| 220 | print ("trybot_image: %s" % trybot_image) |
| 221 | print ("vanilla_image: %s" % vanilla_image) |
cmtice | d54f980 | 2015-02-05 11:04:11 -0800 | [diff] [blame] | 222 | if len(trybot_image) == 0: |
| 223 | self._l.LogError("Unable to find trybot_image for %s!" % description) |
| 224 | return 1 |
| 225 | if len(vanilla_image) == 0: |
| 226 | self._l.LogError("Unable to find vanilla image for %s!" % description) |
| 227 | return 1 |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 228 | if os.getlogin() == ROLE_ACCOUNT: |
| 229 | self._FinishSetup() |
| 230 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 231 | self._TestImages(trybot_image, vanilla_image) |
| 232 | self._SendEmail() |
| 233 | # Only try to copy the image files if the test runs ran successfully. |
| 234 | self._CopyWeeklyReportFiles(trybot_image, vanilla_image) |
cmtice | 46093e5 | 2014-12-09 14:59:16 -0800 | [diff] [blame] | 235 | return 0 |
| 236 | |
| 237 | |
| 238 | def Main(argv): |
| 239 | """The main function.""" |
| 240 | |
| 241 | # Common initializations |
| 242 | command_executer.InitCommandExecuter() |
| 243 | parser = optparse.OptionParser() |
| 244 | parser.add_option("--remote", |
| 245 | dest="remote", |
| 246 | help="Remote machines to run tests on.") |
| 247 | parser.add_option("--board", |
| 248 | dest="board", |
| 249 | default="x86-zgb", |
| 250 | help="The target board.") |
| 251 | parser.add_option("--chromeos_root", |
| 252 | dest="chromeos_root", |
| 253 | help="The chromeos root from which to run tests.") |
| 254 | parser.add_option("--weekday", default="", |
| 255 | dest="weekday", |
| 256 | help="The day of the week for which to run tests.") |
| 257 | options, _ = parser.parse_args(argv) |
| 258 | if not options.board: |
| 259 | print "Please give a board." |
| 260 | return 1 |
| 261 | if not options.remote: |
| 262 | print "Please give at least one remote machine." |
| 263 | return 1 |
| 264 | if not options.chromeos_root: |
| 265 | print "Please specify the ChromeOS root directory." |
| 266 | return 1 |
| 267 | |
| 268 | fc = ToolchainComparator(options.board, options.remote, |
| 269 | options.chromeos_root, options.weekday) |
| 270 | return fc.DoAll() |
| 271 | |
| 272 | |
| 273 | if __name__ == "__main__": |
| 274 | retval = Main(sys.argv) |
| 275 | sys.exit(retval) |