asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Script to test different toolchains against ChromeOS benchmarks. |
| 4 | import optparse |
| 5 | import os |
| 6 | import sys |
| 7 | import build_chromeos |
| 8 | import setup_chromeos |
| 9 | from utils import command_executer |
| 10 | from utils import misc |
| 11 | from utils import logger |
| 12 | |
| 13 | |
| 14 | class GCCConfig(object): |
| 15 | def __init__(self, githash): |
| 16 | self.githash = githash |
| 17 | |
| 18 | |
| 19 | class ToolchainConfig: |
| 20 | def __init__(self, gcc_config=None, binutils_config=None): |
| 21 | self.gcc_config = gcc_config |
| 22 | |
| 23 | |
| 24 | class ChromeOSCheckout(object): |
| 25 | def __init__(self, board, chromeos_root): |
| 26 | self._board = board |
| 27 | self._chromeos_root = chromeos_root |
| 28 | self._ce = command_executer.GetCommandExecuter() |
| 29 | self._l = logger.GetLogger() |
| 30 | |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 31 | def _DeleteChroot(self): |
asharif | 8741427 | 2013-02-19 19:58:45 +0000 | [diff] [blame] | 32 | command = "cd %s; cros_sdk --delete" % self._chromeos_root |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 33 | return self._ce.RunCommand(command) |
| 34 | |
asharif | 6797358 | 2013-02-19 20:19:40 +0000 | [diff] [blame] | 35 | def _DeleteCcahe(self): |
| 36 | # crosbug.com/34956 |
| 37 | command = "sudo rm -rf %s" % os.path.join(self._chromeos_root, ".cache") |
| 38 | return self._ce.RunCommand(command) |
| 39 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 40 | def _BuildAndImage(self, label=""): |
| 41 | if (not label or |
| 42 | not misc.DoesLabelExist(self._chromeos_root, self._board, label)): |
| 43 | build_chromeos_args = [build_chromeos.__file__, |
| 44 | "--chromeos_root=%s" % self._chromeos_root, |
| 45 | "--board=%s" % self._board, |
| 46 | "--rebuild"] |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 47 | if self._public: |
| 48 | build_chromeos_args.append("--env=USE=-chrome_internal") |
Yunlian Jiang | d145a58 | 2013-08-19 13:59:34 -0700 | [diff] [blame] | 49 | if label == "vanilla": |
| 50 | build_chromeos_args.append("--vanilla_image") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 51 | ret = build_chromeos.Main(build_chromeos_args) |
| 52 | if ret: |
| 53 | raise Exception("Couldn't build ChromeOS!") |
| 54 | if label: |
| 55 | misc.LabelLatestImage(self._chromeos_root, self._board, label) |
| 56 | return label |
| 57 | |
| 58 | def _SetupBoard(self, env_dict): |
| 59 | env_string = misc.GetEnvStringFromDict(env_dict) |
| 60 | command = ("%s %s" % |
| 61 | (env_string, |
| 62 | misc.GetSetupBoardCommand(self._board, |
| 63 | usepkg=False))) |
| 64 | ret = self._ce.ChrootRunCommand(self._chromeos_root, |
| 65 | command) |
| 66 | assert ret == 0, "Could not setup board with new toolchain." |
| 67 | |
| 68 | def _UnInstallToolchain(self): |
| 69 | command = ("sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc" % |
| 70 | misc.GetCtargetFromBoard(self._board, |
| 71 | self._chromeos_root)) |
| 72 | ret = self._ce.ChrootRunCommand(self._chromeos_root, |
| 73 | command) |
| 74 | if ret: |
| 75 | raise Exception("Couldn't uninstall the toolchain!") |
| 76 | |
| 77 | def _CheckoutChromeOS(self): |
| 78 | # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot). |
| 79 | if not os.path.exists(self._chromeos_root): |
| 80 | setup_chromeos_args = [setup_chromeos.__file__, |
Caroline Tice | c35909e | 2013-10-02 17:13:13 -0700 | [diff] [blame] | 81 | "--dir=%s" % self._chromeos_root] |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 82 | if self._public: |
| 83 | setup_chromeos_args.append("--public") |
asharif | 5fe40e2 | 2013-02-19 19:58:50 +0000 | [diff] [blame] | 84 | ret = setup_chromeos.Main(setup_chromeos_args) |
| 85 | if ret: |
| 86 | raise Exception("Couldn't run setup_chromeos!") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 87 | |
Caroline Tice | c35909e | 2013-10-02 17:13:13 -0700 | [diff] [blame] | 88 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 89 | def _BuildToolchain(self, config): |
| 90 | self._UnInstallToolchain() |
| 91 | self._SetupBoard({"USE": "git_gcc", |
| 92 | "GCC_GITHASH": config.gcc_config.githash, |
| 93 | "EMERGE_DEFAULT_OPTS": "--exclude=gcc"}) |
| 94 | |
| 95 | |
| 96 | class ToolchainComparator(ChromeOSCheckout): |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 97 | def __init__(self, board, remotes, configs, clean, public, force_mismatch): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 98 | self._board = board |
| 99 | self._remotes = remotes |
| 100 | self._chromeos_root = "chromeos" |
| 101 | self._configs = configs |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 102 | self._clean = clean |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 103 | self._public = public |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 104 | self._force_mismatch = force_mismatch |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 105 | self._ce = command_executer.GetCommandExecuter() |
| 106 | self._l = logger.GetLogger() |
| 107 | ChromeOSCheckout.__init__(self, board, self._chromeos_root) |
| 108 | |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 109 | |
| 110 | def _FinishSetup(self): |
| 111 | # Get correct .boto file |
| 112 | current_dir = os.getcwd() |
| 113 | src = "/home/mobiletc-prebuild/.boto" |
| 114 | dest = os.path.join(current_dir, self._chromeos_root, |
| 115 | "src/private-overlays/chromeos-overlay/" |
| 116 | "googlestorage_account.boto") |
| 117 | # Copy the file to the correct place |
| 118 | copy_cmd = "cp %s %s" % (src, dest) |
| 119 | retval = self._ce.RunCommand(copy_cmd) |
| 120 | if retval: |
| 121 | raise Exception("Couldn't copy .boto file for google storage.") |
| 122 | |
| 123 | # Fix protections on ssh key |
| 124 | command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target" |
| 125 | "/chrome-src-internal/src/third_party/chromite/ssh_keys" |
| 126 | "/testing_rsa") |
| 127 | retval = self._ce.ChrootRunCommand(self._chromeos_root, command) |
| 128 | if retval: |
| 129 | raise Exception("chmod for testing_rsa failed") |
| 130 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 131 | def _TestLabels(self, labels): |
| 132 | experiment_file = "toolchain_experiment.txt" |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 133 | image_args = "" |
| 134 | if self._force_mismatch: |
asharif | 0b5d5c8 | 2013-02-19 20:42:56 +0000 | [diff] [blame] | 135 | image_args = "--force-mismatch" |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 136 | experiment_header = """ |
| 137 | board: %s |
| 138 | remote: %s |
| 139 | """ % (self._board, self._remotes) |
| 140 | experiment_tests = """ |
cmtice | 0440388 | 2013-11-04 16:38:37 -0500 | [diff] [blame] | 141 | benchmark: all_perfv2 { |
| 142 | suite: telemetry_Crosperf |
cmtice | 6de7f8f | 2014-03-14 14:08:21 -0700 | [diff] [blame] | 143 | iterations: 3 |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 144 | } |
| 145 | """ |
| 146 | with open(experiment_file, "w") as f: |
| 147 | print >>f, experiment_header |
| 148 | print >>f, experiment_tests |
| 149 | for label in labels: |
| 150 | # TODO(asharif): Fix crosperf so it accepts labels with symbols |
| 151 | crosperf_label = label |
| 152 | crosperf_label = crosperf_label.replace("-", "minus") |
| 153 | crosperf_label = crosperf_label.replace("+", "plus") |
asharif | e4a5a8f | 2013-02-19 20:19:33 +0000 | [diff] [blame] | 154 | crosperf_label = crosperf_label.replace(".", "") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 155 | experiment_image = """ |
| 156 | %s { |
| 157 | chromeos_image: %s |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 158 | image_args: %s |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 159 | } |
| 160 | """ % (crosperf_label, |
| 161 | os.path.join(misc.GetImageDir(self._chromeos_root, self._board), |
| 162 | label, |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 163 | "chromiumos_test_image.bin"), |
| 164 | image_args) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 165 | print >>f, experiment_image |
| 166 | crosperf = os.path.join(os.path.dirname(__file__), |
| 167 | "crosperf", |
| 168 | "crosperf") |
asharif | b2cd634 | 2013-02-19 19:42:57 +0000 | [diff] [blame] | 169 | command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 170 | ret = self._ce.RunCommand(command) |
| 171 | if ret: |
| 172 | raise Exception("Couldn't run crosperf!") |
| 173 | |
| 174 | def DoAll(self): |
| 175 | self._CheckoutChromeOS() |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 176 | labels = [] |
| 177 | vanilla_label = self._BuildAndImage("vanilla") |
| 178 | labels.append(vanilla_label) |
| 179 | for config in self._configs: |
| 180 | label = misc.GetFilenameFromString(config.gcc_config.githash) |
| 181 | if (not misc.DoesLabelExist(self._chromeos_root, |
| 182 | self._board, |
| 183 | label)): |
| 184 | self._BuildToolchain(config) |
| 185 | label = self._BuildAndImage(label) |
| 186 | labels.append(label) |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 187 | self._FinishSetup() |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 188 | self._TestLabels(labels) |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 189 | if self._clean: |
| 190 | ret = self._DeleteChroot() |
| 191 | if ret: return ret |
asharif | 6797358 | 2013-02-19 20:19:40 +0000 | [diff] [blame] | 192 | ret = self._DeleteCcahe() |
| 193 | if ret: return ret |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 194 | return 0 |
| 195 | |
| 196 | |
| 197 | def Main(argv): |
| 198 | """The main function.""" |
| 199 | # Common initializations |
| 200 | ### command_executer.InitCommandExecuter(True) |
| 201 | command_executer.InitCommandExecuter() |
| 202 | parser = optparse.OptionParser() |
| 203 | parser.add_option("--remote", |
| 204 | dest="remote", |
| 205 | help="Remote machines to run tests on.") |
| 206 | parser.add_option("--board", |
| 207 | dest="board", |
| 208 | default="x86-zgb", |
| 209 | help="The target board.") |
| 210 | parser.add_option("--githashes", |
| 211 | dest="githashes", |
| 212 | default="master", |
| 213 | help="The gcc githashes to test.") |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 214 | parser.add_option("--clean", |
| 215 | dest="clean", |
| 216 | default=False, |
| 217 | action="store_true", |
| 218 | help="Clean the chroot after testing.") |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 219 | parser.add_option("--public", |
| 220 | dest="public", |
| 221 | default=False, |
| 222 | action="store_true", |
| 223 | help="Use the public checkout/build.") |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 224 | parser.add_option("--force-mismatch", |
| 225 | dest="force_mismatch", |
| 226 | default="", |
| 227 | help="Force the image regardless of board mismatch") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 228 | options, _ = parser.parse_args(argv) |
| 229 | if not options.board: |
| 230 | print "Please give a board." |
| 231 | return 1 |
| 232 | if not options.remote: |
| 233 | print "Please give at least one remote machine." |
| 234 | return 1 |
| 235 | toolchain_configs = [] |
| 236 | for githash in options.githashes.split(","): |
| 237 | gcc_config = GCCConfig(githash=githash) |
| 238 | toolchain_config = ToolchainConfig(gcc_config=gcc_config) |
| 239 | toolchain_configs.append(toolchain_config) |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 240 | fc = ToolchainComparator(options.board, options.remote, toolchain_configs, |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 241 | options.clean, options.public, |
| 242 | options.force_mismatch) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 243 | return fc.DoAll() |
| 244 | |
| 245 | |
| 246 | if __name__ == "__main__": |
| 247 | retval = Main(sys.argv) |
| 248 | sys.exit(retval) |