Caroline Tice | 4bd7046 | 2016-10-05 15:41:13 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 2 | """Script for running llvm validation tests on ChromeOS. |
| 3 | |
| 4 | This script launches a buildbot to build ChromeOS with the llvm on |
| 5 | a particular board; then it finds and downloads the trybot image and the |
| 6 | corresponding official image, and runs test for correctness. |
| 7 | It then generates a report, emails it to the c-compiler-chrome, as |
| 8 | well as copying the result into a directory. |
| 9 | """ |
| 10 | |
| 11 | # Script to test different toolchains against ChromeOS benchmarks. |
| 12 | |
| 13 | from __future__ import print_function |
| 14 | |
| 15 | import argparse |
| 16 | import datetime |
| 17 | import os |
| 18 | import sys |
| 19 | import time |
| 20 | |
Caroline Tice | a8af9a7 | 2016-07-20 12:52:59 -0700 | [diff] [blame] | 21 | from cros_utils import command_executer |
| 22 | from cros_utils import logger |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 23 | |
Caroline Tice | a8af9a7 | 2016-07-20 12:52:59 -0700 | [diff] [blame] | 24 | from cros_utils import buildbot_utils |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 25 | |
| 26 | # CL that uses LLVM to build the peppy image. |
| 27 | USE_LLVM_PATCH = '295217' |
| 28 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 29 | CROSTC_ROOT = '/usr/local/google/crostc' |
| 30 | ROLE_ACCOUNT = 'mobiletc-prebuild' |
| 31 | TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
| 32 | MAIL_PROGRAM = '~/var/bin/mail-sheriff' |
| 33 | VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result') |
| 34 | START_DATE = datetime.date(2016, 1, 1) |
| 35 | TEST_PER_DAY = 2 |
| 36 | TEST_BOARD = [ |
Caroline Tice | a8517eb | 2016-11-30 14:52:17 -0800 | [diff] [blame] | 37 | 'squawks', # x86_64, rambi (baytrail) |
| 38 | 'terra', # x86_64, strago (braswell) |
| 39 | 'lulu', # x86_64, auron (broadwell) |
| 40 | 'peach_pit', # arm, peach (exynos-5420) |
| 41 | 'peppy', # x86_64, slippy (haswell celeron) |
| 42 | 'link', # x86_64, ivybridge (ivybridge) |
| 43 | 'nyan_big', # arm, nyan (tegra) |
| 44 | 'sentry', # x86_64, kunimitsu (skylake-u) |
| 45 | 'chell', # x86_64, glados (skylake-y) |
| 46 | 'daisy', # arm, daisy (exynos) |
Yunlian Jiang | 9e32aa6 | 2017-01-27 13:15:41 -0800 | [diff] [blame] | 47 | 'caroline', # amd64 |
| 48 | 'kevin', # arm, gru (Rockchip) |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 49 | ] |
| 50 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 51 | |
| 52 | class ToolchainVerifier(object): |
| 53 | """Class for the toolchain verifier.""" |
| 54 | |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 55 | def __init__(self, board, chromeos_root, weekday, patches, compiler): |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 56 | self._board = board |
| 57 | self._chromeos_root = chromeos_root |
| 58 | self._base_dir = os.getcwd() |
| 59 | self._ce = command_executer.GetCommandExecuter() |
| 60 | self._l = logger.GetLogger() |
Caroline Tice | 314ea56 | 2016-06-24 15:59:01 -0700 | [diff] [blame] | 61 | self._compiler = compiler |
Caroline Tice | 4bd7046 | 2016-10-05 15:41:13 -0700 | [diff] [blame] | 62 | self._build = '%s-%s-toolchain' % (board, compiler) |
Caroline Tice | de60077 | 2016-10-18 15:27:51 -0700 | [diff] [blame] | 63 | self._patches = patches.split(',') if patches else [] |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 64 | self._patches_string = '_'.join(str(p) for p in self._patches) |
| 65 | |
| 66 | if not weekday: |
| 67 | self._weekday = time.strftime('%a') |
| 68 | else: |
| 69 | self._weekday = weekday |
Caroline Tice | d00ad41 | 2016-07-02 18:00:18 -0700 | [diff] [blame] | 70 | self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 71 | |
| 72 | def _FinishSetup(self): |
| 73 | """Make sure testing_rsa file is properly set up.""" |
| 74 | # Fix protections on ssh key |
| 75 | command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' |
| 76 | '/chrome-src-internal/src/third_party/chromite/ssh_keys' |
| 77 | '/testing_rsa') |
| 78 | ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) |
| 79 | if ret_val != 0: |
| 80 | raise RuntimeError('chmod for testing_rsa failed') |
| 81 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 82 | def DoAll(self): |
| 83 | """Main function inside ToolchainComparator class. |
| 84 | |
| 85 | Launch trybot, get image names, create crosperf experiment file, run |
| 86 | crosperf, and copy images into seven-day report directories. |
| 87 | """ |
Caroline Tice | 1ba6d57 | 2016-10-10 11:31:54 -0700 | [diff] [blame] | 88 | flags = ['--hwtest'] |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 89 | date_str = datetime.date.today() |
| 90 | description = 'master_%s_%s_%s' % (self._patches_string, self._build, |
| 91 | date_str) |
Caroline Tice | 0974197 | 2016-11-02 15:22:28 -0700 | [diff] [blame] | 92 | _ = buildbot_utils.GetTrybotImage( |
Caroline Tice | 1ba6d57 | 2016-10-10 11:31:54 -0700 | [diff] [blame] | 93 | self._chromeos_root, |
| 94 | self._build, |
| 95 | self._patches, |
| 96 | description, |
Caroline Tice | 0974197 | 2016-11-02 15:22:28 -0700 | [diff] [blame] | 97 | other_flags=flags, |
| 98 | async=True) |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 99 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 100 | return 0 |
| 101 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 102 | def Main(argv): |
| 103 | """The main function.""" |
| 104 | |
| 105 | # Common initializations |
| 106 | command_executer.InitCommandExecuter() |
| 107 | parser = argparse.ArgumentParser() |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 108 | parser.add_argument( |
| 109 | '--chromeos_root', |
| 110 | dest='chromeos_root', |
| 111 | help='The chromeos root from which to run tests.') |
| 112 | parser.add_argument( |
| 113 | '--weekday', |
| 114 | default='', |
| 115 | dest='weekday', |
| 116 | help='The day of the week for which to run tests.') |
| 117 | parser.add_argument( |
| 118 | '--board', default='', dest='board', help='The board to test.') |
| 119 | parser.add_argument( |
| 120 | '--patch', |
| 121 | dest='patches', |
Caroline Tice | de60077 | 2016-10-18 15:27:51 -0700 | [diff] [blame] | 122 | default='', |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 123 | help='The patches to use for the testing, ' |
| 124 | "seprate the patch numbers with ',' " |
| 125 | 'for more than one patches.') |
| 126 | parser.add_argument( |
| 127 | '--compiler', |
| 128 | dest='compiler', |
Caroline Tice | 4bd7046 | 2016-10-05 15:41:13 -0700 | [diff] [blame] | 129 | help='Which compiler (llvm, llvm-next or gcc) to use for ' |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 130 | 'testing.') |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 131 | |
| 132 | options = parser.parse_args(argv[1:]) |
| 133 | if not options.chromeos_root: |
| 134 | print('Please specify the ChromeOS root directory.') |
| 135 | return 1 |
Caroline Tice | 314ea56 | 2016-06-24 15:59:01 -0700 | [diff] [blame] | 136 | if not options.compiler: |
Caroline Tice | 4bd7046 | 2016-10-05 15:41:13 -0700 | [diff] [blame] | 137 | print('Please specify which compiler to test (gcc, llvm, or llvm-next).') |
Caroline Tice | 314ea56 | 2016-06-24 15:59:01 -0700 | [diff] [blame] | 138 | return 1 |
Caroline Tice | de60077 | 2016-10-18 15:27:51 -0700 | [diff] [blame] | 139 | patches = options.patches |
| 140 | if not patches and options.compiler == 'llvm': |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 141 | patches = USE_LLVM_PATCH |
| 142 | |
| 143 | if options.board: |
| 144 | fv = ToolchainVerifier(options.board, options.chromeos_root, |
Caroline Tice | 314ea56 | 2016-06-24 15:59:01 -0700 | [diff] [blame] | 145 | options.weekday, patches, options.compiler) |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 146 | return fv.Doall() |
| 147 | |
| 148 | today = datetime.date.today() |
| 149 | delta = today - START_DATE |
| 150 | days = delta.days |
| 151 | |
| 152 | start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) |
| 153 | for i in range(TEST_PER_DAY): |
Yunlian Jiang | 54e72b3 | 2016-06-21 14:13:03 -0700 | [diff] [blame] | 154 | try: |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 155 | board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] |
| 156 | fv = ToolchainVerifier(board, options.chromeos_root, options.weekday, |
| 157 | patches, options.compiler) |
Yunlian Jiang | 54e72b3 | 2016-06-21 14:13:03 -0700 | [diff] [blame] | 158 | fv.DoAll() |
| 159 | except SystemExit: |
Caroline Tice | d00ad41 | 2016-07-02 18:00:18 -0700 | [diff] [blame] | 160 | logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board) |
Yunlian Jiang | 54e72b3 | 2016-06-21 14:13:03 -0700 | [diff] [blame] | 161 | with open(logfile, 'w') as f: |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 162 | f.write('Verifier got an exception, please check the log.\n') |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 163 | |
Caroline Tice | a12e974 | 2016-09-08 13:35:02 -0700 | [diff] [blame] | 164 | |
Yunlian Jiang | c571337 | 2016-06-15 11:37:50 -0700 | [diff] [blame] | 165 | if __name__ == '__main__': |
| 166 | retval = Main(sys.argv) |
| 167 | sys.exit(retval) |