Caroline Tice | f6ef439 | 2017-04-06 17:16:05 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 2 | |
| 3 | # Script to test different toolchains against ChromeOS benchmarks. |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 4 | """Toolchain team nightly performance test script (local builds).""" |
| 5 | |
| 6 | from __future__ import print_function |
| 7 | |
| 8 | import argparse |
cmtice | 7cdb11a | 2015-05-28 10:24:41 -0700 | [diff] [blame] | 9 | import datetime |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 10 | import os |
| 11 | import sys |
| 12 | import build_chromeos |
| 13 | import setup_chromeos |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 14 | from cros_utils import command_executer |
| 15 | from cros_utils import misc |
| 16 | from cros_utils import logger |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 17 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 18 | CROSTC_ROOT = '/usr/local/google/crostc' |
| 19 | MAIL_PROGRAM = '~/var/bin/mail-sheriff' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 20 | PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') |
| 21 | NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 22 | |
cmtice | 94bc470 | 2014-05-29 16:29:04 -0700 | [diff] [blame] | 23 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 24 | class GCCConfig(object): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 25 | """GCC configuration class.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 26 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 27 | def __init__(self, githash): |
| 28 | self.githash = githash |
| 29 | |
| 30 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 31 | class ToolchainConfig(object): |
| 32 | """Toolchain configuration class.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 33 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 34 | def __init__(self, gcc_config=None): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 35 | self.gcc_config = gcc_config |
| 36 | |
| 37 | |
| 38 | class ChromeOSCheckout(object): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 39 | """Main class for checking out, building and testing ChromeOS.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 40 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 41 | def __init__(self, board, chromeos_root): |
| 42 | self._board = board |
| 43 | self._chromeos_root = chromeos_root |
| 44 | self._ce = command_executer.GetCommandExecuter() |
| 45 | self._l = logger.GetLogger() |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 46 | self._build_num = None |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 47 | |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 48 | def _DeleteChroot(self): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 49 | command = 'cd %s; cros_sdk --delete' % self._chromeos_root |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 50 | return self._ce.RunCommand(command) |
| 51 | |
asharif | 6797358 | 2013-02-19 20:19:40 +0000 | [diff] [blame] | 52 | def _DeleteCcahe(self): |
| 53 | # crosbug.com/34956 |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 54 | command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache') |
asharif | 6797358 | 2013-02-19 20:19:40 +0000 | [diff] [blame] | 55 | return self._ce.RunCommand(command) |
| 56 | |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 57 | def _GetBuildNumber(self): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 58 | """Get the build number of the ChromeOS image from the chroot. |
| 59 | |
| 60 | This function assumes a ChromeOS image has been built in the chroot. |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 61 | It translates the 'latest' symlink in the |
| 62 | <chroot>/src/build/images/<board> directory, to find the actual |
| 63 | ChromeOS build number for the image that was built. For example, if |
| 64 | src/build/image/lumpy/latest -> R37-5982.0.2014_06_23_0454-a1, then |
| 65 | This function would parse it out and assign 'R37-5982' to self._build_num. |
| 66 | This is used to determine the official, vanilla build to use for |
| 67 | comparison tests. |
| 68 | """ |
| 69 | # Get the path to 'latest' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 70 | sym_path = os.path.join( |
| 71 | misc.GetImageDir(self._chromeos_root, self._board), 'latest') |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 72 | # Translate the symlink to its 'real' path. |
| 73 | real_path = os.path.realpath(sym_path) |
| 74 | # Break up the path and get the last piece |
| 75 | # (e.g. 'R37-5982.0.2014_06_23_0454-a1" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 76 | path_pieces = real_path.split('/') |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 77 | last_piece = path_pieces[-1] |
| 78 | # Break this piece into the image number + other pieces, and get the |
| 79 | # image number [ 'R37-5982', '0', '2014_06_23_0454-a1'] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 80 | image_parts = last_piece.split('.') |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 81 | self._build_num = image_parts[0] |
| 82 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 83 | def _BuildLabelName(self, config): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 84 | pieces = config.split('/') |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 85 | compiler_version = pieces[-1] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 86 | label = compiler_version + '_tot_afdo' |
Caroline Tice | 80eab98 | 2015-11-04 14:03:14 -0800 | [diff] [blame] | 87 | return label |
| 88 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 89 | def _BuildAndImage(self, label=''): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 90 | if (not label or |
| 91 | not misc.DoesLabelExist(self._chromeos_root, self._board, label)): |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 92 | build_chromeos_args = [ |
| 93 | build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root, |
| 94 | '--board=%s' % self._board, '--rebuild' |
| 95 | ] |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 96 | if self._public: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 97 | build_chromeos_args.append('--env=USE=-chrome_internal') |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 98 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 99 | ret = build_chromeos.Main(build_chromeos_args) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 100 | if ret != 0: |
| 101 | raise RuntimeError("Couldn't build ChromeOS!") |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 102 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 103 | if not self._build_num: |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 104 | self._GetBuildNumber() |
| 105 | # Check to see if we need to create the symbolic link for the vanilla |
| 106 | # image, and do so if appropriate. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 107 | if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'): |
| 108 | build_name = '%s-release/%s.0.0' % (self._board, self._build_num) |
| 109 | full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root, |
| 110 | 'chroot/tmp', build_name) |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 111 | misc.LabelLatestImage(self._chromeos_root, self._board, label, |
| 112 | full_vanilla_path) |
| 113 | else: |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 114 | misc.LabelLatestImage(self._chromeos_root, self._board, label) |
| 115 | return label |
| 116 | |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 117 | def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 118 | env_string = misc.GetEnvStringFromDict(env_dict) |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 119 | command = ('%s %s' % (env_string, misc.GetSetupBoardCommand( |
| 120 | self._board, usepkg=usepkg_flag, force=clobber_flag))) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 121 | ret = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 122 | error_str = "Could not setup board: '%s'" % command |
| 123 | assert ret == 0, error_str |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 124 | |
| 125 | def _UnInstallToolchain(self): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 126 | command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' % |
| 127 | misc.GetCtargetFromBoard(self._board, self._chromeos_root)) |
| 128 | ret = self._ce.ChrootRunCommand(self._chromeos_root, command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 129 | if ret != 0: |
| 130 | raise RuntimeError("Couldn't uninstall the toolchain!") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 131 | |
| 132 | def _CheckoutChromeOS(self): |
| 133 | # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot). |
| 134 | if not os.path.exists(self._chromeos_root): |
Rahul Chaudhry | 4d4565e | 2016-01-27 10:46:09 -0800 | [diff] [blame] | 135 | setup_chromeos_args = ['--dir=%s' % self._chromeos_root] |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 136 | if self._public: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 137 | setup_chromeos_args.append('--public') |
asharif | 5fe40e2 | 2013-02-19 19:58:50 +0000 | [diff] [blame] | 138 | ret = setup_chromeos.Main(setup_chromeos_args) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 139 | if ret != 0: |
| 140 | raise RuntimeError("Couldn't run setup_chromeos!") |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 141 | |
| 142 | def _BuildToolchain(self, config): |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 143 | # Call setup_board for basic, vanilla setup. |
| 144 | self._SetupBoard({}, usepkg_flag=True, clobber_flag=False) |
| 145 | # Now uninstall the vanilla compiler and setup/build our custom |
| 146 | # compiler. |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 147 | self._UnInstallToolchain() |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 148 | envdict = { |
| 149 | 'USE': 'git_gcc', |
| 150 | 'GCC_GITHASH': config.gcc_config.githash, |
| 151 | 'EMERGE_DEFAULT_OPTS': '--exclude=gcc' |
| 152 | } |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 153 | self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 154 | |
| 155 | |
| 156 | class ToolchainComparator(ChromeOSCheckout): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 157 | """Main class for running tests and generating reports.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 158 | |
| 159 | def __init__(self, |
| 160 | board, |
| 161 | remotes, |
| 162 | configs, |
| 163 | clean, |
| 164 | public, |
| 165 | force_mismatch, |
| 166 | noschedv2=False): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 167 | self._board = board |
| 168 | self._remotes = remotes |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 169 | self._chromeos_root = 'chromeos' |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 170 | self._configs = configs |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 171 | self._clean = clean |
asharif | e6b72fe | 2013-02-19 19:58:18 +0000 | [diff] [blame] | 172 | self._public = public |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 173 | self._force_mismatch = force_mismatch |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 174 | self._ce = command_executer.GetCommandExecuter() |
| 175 | self._l = logger.GetLogger() |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 176 | timestamp = datetime.datetime.strftime(datetime.datetime.now(), |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 177 | '%Y-%m-%d_%H:%M:%S') |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 178 | self._reports_dir = os.path.join( |
| 179 | NIGHTLY_TESTS_DIR, |
| 180 | '%s.%s' % (timestamp, board),) |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 181 | self._noschedv2 = noschedv2 |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 182 | ChromeOSCheckout.__init__(self, board, self._chromeos_root) |
| 183 | |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 184 | def _FinishSetup(self): |
| 185 | # Get correct .boto file |
| 186 | current_dir = os.getcwd() |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 187 | src = '/usr/local/google/home/mobiletc-prebuild/.boto' |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 188 | dest = os.path.join(current_dir, self._chromeos_root, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 189 | 'src/private-overlays/chromeos-overlay/' |
| 190 | 'googlestorage_account.boto') |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 191 | # Copy the file to the correct place |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 192 | copy_cmd = 'cp %s %s' % (src, dest) |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 193 | retv = self._ce.RunCommand(copy_cmd) |
| 194 | if retv != 0: |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 195 | raise RuntimeError("Couldn't copy .boto file for google storage.") |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 196 | |
| 197 | # Fix protections on ssh key |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 198 | command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' |
| 199 | '/chrome-src-internal/src/third_party/chromite/ssh_keys' |
| 200 | '/testing_rsa') |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 201 | retv = self._ce.ChrootRunCommand(self._chromeos_root, command) |
| 202 | if retv != 0: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 203 | raise RuntimeError('chmod for testing_rsa failed') |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 204 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 205 | def _TestLabels(self, labels): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 206 | experiment_file = 'toolchain_experiment.txt' |
| 207 | image_args = '' |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 208 | if self._force_mismatch: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 209 | image_args = '--force-mismatch' |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 210 | experiment_header = """ |
| 211 | board: %s |
| 212 | remote: %s |
cmtice | d1f03b8 | 2015-06-30 15:19:23 -0700 | [diff] [blame] | 213 | retries: 1 |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 214 | """ % (self._board, self._remotes) |
| 215 | experiment_tests = """ |
cmtice | 0c84ea7 | 2015-06-25 14:22:36 -0700 | [diff] [blame] | 216 | benchmark: all_toolchain_perf { |
cmtice | 0440388 | 2013-11-04 16:38:37 -0500 | [diff] [blame] | 217 | suite: telemetry_Crosperf |
cmtice | 6de7f8f | 2014-03-14 14:08:21 -0700 | [diff] [blame] | 218 | iterations: 3 |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 219 | } |
| 220 | """ |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 221 | |
| 222 | with open(experiment_file, 'w') as f: |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 223 | f.write(experiment_header) |
| 224 | f.write(experiment_tests) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 225 | for label in labels: |
| 226 | # TODO(asharif): Fix crosperf so it accepts labels with symbols |
| 227 | crosperf_label = label |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 228 | crosperf_label = crosperf_label.replace('-', '_') |
| 229 | crosperf_label = crosperf_label.replace('+', '_') |
| 230 | crosperf_label = crosperf_label.replace('.', '') |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 231 | |
| 232 | # Use the official build instead of building vanilla ourselves. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 233 | if label == 'vanilla': |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 234 | build_name = '%s-release/%s.0.0' % (self._board, self._build_num) |
| 235 | |
| 236 | # Now add 'official build' to test file. |
| 237 | official_image = """ |
| 238 | official_image { |
| 239 | chromeos_root: %s |
| 240 | build: %s |
| 241 | } |
| 242 | """ % (self._chromeos_root, build_name) |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 243 | f.write(official_image) |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 244 | |
cmtice | 94bc470 | 2014-05-29 16:29:04 -0700 | [diff] [blame] | 245 | else: |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 246 | experiment_image = """ |
| 247 | %s { |
| 248 | chromeos_image: %s |
| 249 | image_args: %s |
| 250 | } |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 251 | """ % (crosperf_label, os.path.join( |
| 252 | misc.GetImageDir(self._chromeos_root, self._board), label, |
| 253 | 'chromiumos_test_image.bin'), image_args) |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 254 | f.write(experiment_image) |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 255 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 256 | crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf') |
Han Shen | 4349429 | 2015-09-14 10:26:40 -0700 | [diff] [blame] | 257 | noschedv2_opts = '--noschedv2' if self._noschedv2 else '' |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 258 | command = ('{crosperf} --no_email=True --results_dir={r_dir} ' |
| 259 | '--json_report=True {noschedv2_opts} {exp_file}').format( |
| 260 | crosperf=crosperf, |
| 261 | r_dir=self._reports_dir, |
| 262 | noschedv2_opts=noschedv2_opts, |
| 263 | exp_file=experiment_file) |
cmtice | aa700b0 | 2015-06-12 13:26:47 -0700 | [diff] [blame] | 264 | |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 265 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 266 | if ret != 0: |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 267 | raise RuntimeError('Crosperf execution error!') |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 268 | else: |
| 269 | # Copy json report to pending archives directory. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 270 | command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) |
Caroline Tice | ebbc3da | 2015-09-03 10:27:20 -0700 | [diff] [blame] | 271 | ret = self._ce.RunCommand(command) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 272 | return |
cmtice | 56fb716 | 2014-06-18 11:32:15 -0700 | [diff] [blame] | 273 | |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 274 | def _SendEmail(self): |
| 275 | """Find email msesage generated by crosperf and send it.""" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 276 | filename = os.path.join(self._reports_dir, 'msg_body.html') |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 277 | if (os.path.exists(filename) and |
| 278 | os.path.exists(os.path.expanduser(MAIL_PROGRAM))): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 279 | command = ('cat %s | %s -s "Nightly test results, %s" -team -html' % |
| 280 | (filename, MAIL_PROGRAM, self._board)) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 281 | self._ce.RunCommand(command) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 282 | |
| 283 | def DoAll(self): |
| 284 | self._CheckoutChromeOS() |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 285 | labels = [] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 286 | labels.append('vanilla') |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 287 | for config in self._configs: |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 288 | label = self._BuildLabelName(config.gcc_config.githash) |
| 289 | if not misc.DoesLabelExist(self._chromeos_root, self._board, label): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 290 | self._BuildToolchain(config) |
| 291 | label = self._BuildAndImage(label) |
| 292 | labels.append(label) |
cmtice | a6255d0 | 2014-01-10 10:27:22 -0800 | [diff] [blame] | 293 | self._FinishSetup() |
cmtice | b458809 | 2015-05-27 08:07:50 -0700 | [diff] [blame] | 294 | self._TestLabels(labels) |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 295 | self._SendEmail() |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 296 | if self._clean: |
| 297 | ret = self._DeleteChroot() |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 298 | if ret != 0: |
| 299 | return ret |
asharif | 6797358 | 2013-02-19 20:19:40 +0000 | [diff] [blame] | 300 | ret = self._DeleteCcahe() |
cmtice | 7f3190b | 2015-05-22 14:14:51 -0700 | [diff] [blame] | 301 | if ret != 0: |
| 302 | return ret |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 303 | return 0 |
| 304 | |
| 305 | |
| 306 | def Main(argv): |
| 307 | """The main function.""" |
| 308 | # Common initializations |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 309 | ### command_executer.InitCommandExecuter(True) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 310 | command_executer.InitCommandExecuter() |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 311 | parser = argparse.ArgumentParser() |
Manoj Gupta | aee96b7 | 2016-10-24 13:43:28 -0700 | [diff] [blame] | 312 | parser.add_argument( |
| 313 | '--remote', dest='remote', help='Remote machines to run tests on.') |
| 314 | parser.add_argument( |
| 315 | '--board', dest='board', default='x86-alex', help='The target board.') |
| 316 | parser.add_argument( |
| 317 | '--githashes', |
| 318 | dest='githashes', |
| 319 | default='master', |
| 320 | help='The gcc githashes to test.') |
| 321 | parser.add_argument( |
| 322 | '--clean', |
| 323 | dest='clean', |
| 324 | default=False, |
| 325 | action='store_true', |
| 326 | help='Clean the chroot after testing.') |
| 327 | parser.add_argument( |
| 328 | '--public', |
| 329 | dest='public', |
| 330 | default=False, |
| 331 | action='store_true', |
| 332 | help='Use the public checkout/build.') |
| 333 | parser.add_argument( |
| 334 | '--force-mismatch', |
| 335 | dest='force_mismatch', |
| 336 | default='', |
| 337 | help='Force the image regardless of board mismatch') |
| 338 | parser.add_argument( |
| 339 | '--noschedv2', |
| 340 | dest='noschedv2', |
| 341 | action='store_true', |
| 342 | default=False, |
| 343 | help='Pass --noschedv2 to crosperf.') |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 344 | options = parser.parse_args(argv) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 345 | if not options.board: |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 346 | print('Please give a board.') |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 347 | return 1 |
| 348 | if not options.remote: |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 349 | print('Please give at least one remote machine.') |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 350 | return 1 |
| 351 | toolchain_configs = [] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 352 | for githash in options.githashes.split(','): |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 353 | gcc_config = GCCConfig(githash=githash) |
| 354 | toolchain_config = ToolchainConfig(gcc_config=gcc_config) |
| 355 | toolchain_configs.append(toolchain_config) |
asharif | 3e38de0 | 2013-02-19 19:34:59 +0000 | [diff] [blame] | 356 | fc = ToolchainComparator(options.board, options.remote, toolchain_configs, |
asharif | 58a8c9f | 2013-02-19 20:42:43 +0000 | [diff] [blame] | 357 | options.clean, options.public, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 358 | options.force_mismatch, options.noschedv2) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 359 | return fc.DoAll() |
| 360 | |
| 361 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 362 | if __name__ == '__main__': |
Rahul Chaudhry | 748254e | 2016-01-25 16:49:21 -0800 | [diff] [blame] | 363 | retval = Main(sys.argv[1:]) |
asharif | 96f5969 | 2013-02-16 03:13:36 +0000 | [diff] [blame] | 364 | sys.exit(retval) |